This is an automated email from the ASF dual-hosted git repository. pcongiusti pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 4e2bc7959185de72f32230a1f14255c3860d3999 Author: Pasquale Congiusti <[email protected]> AuthorDate: Thu Dec 7 13:29:02 2023 +0100 feat(pipeline): filter and sort tasks execution --- .../bases/camel.apache.org_integrationkits.yaml | 7 ++ .../camel.apache.org_integrationplatforms.yaml | 14 +++ .../crd/bases/camel.apache.org_integrations.yaml | 7 ++ .../bases/camel.apache.org_kameletbindings.yaml | 8 ++ config/crd/bases/camel.apache.org_pipes.yaml | 8 ++ docs/modules/ROOT/partials/apis/camel-k-crds.adoc | 9 ++ docs/modules/traits/pages/builder.adoc | 6 ++ helm/camel-k/crds/crd-integration-kit.yaml | 7 ++ helm/camel-k/crds/crd-integration-platform.yaml | 14 +++ helm/camel-k/crds/crd-integration.yaml | 7 ++ helm/camel-k/crds/crd-kamelet-binding.yaml | 8 ++ helm/camel-k/crds/crd-pipe.yaml | 8 ++ pkg/apis/camel/v1/trait/builder.go | 4 + pkg/trait/builder.go | 46 ++++++++++ pkg/trait/builder_test.go | 99 ++++++++++++++++++++-- resources/traits.yaml | 6 ++ 16 files changed, 249 insertions(+), 9 deletions(-) diff --git a/config/crd/bases/camel.apache.org_integrationkits.yaml b/config/crd/bases/camel.apache.org_integrationkits.yaml index e3dece12f..3d4714344 100644 --- a/config/crd/bases/camel.apache.org_integrationkits.yaml +++ b/config/crd/bases/camel.apache.org_integrationkits.yaml @@ -284,6 +284,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/config/crd/bases/camel.apache.org_integrationplatforms.yaml b/config/crd/bases/camel.apache.org_integrationplatforms.yaml index f548fe157..240fb4143 100644 --- a/config/crd/bases/camel.apache.org_integrationplatforms.yaml +++ b/config/crd/bases/camel.apache.org_integrationplatforms.yaml @@ -589,6 +589,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. @@ -2472,6 +2479,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/config/crd/bases/camel.apache.org_integrations.yaml b/config/crd/bases/camel.apache.org_integrations.yaml index fd9263a05..fa9fd5c0c 100644 --- a/config/crd/bases/camel.apache.org_integrations.yaml +++ b/config/crd/bases/camel.apache.org_integrations.yaml @@ -6502,6 +6502,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/config/crd/bases/camel.apache.org_kameletbindings.yaml b/config/crd/bases/camel.apache.org_kameletbindings.yaml index 0739d217f..ad82cf8c1 100644 --- a/config/crd/bases/camel.apache.org_kameletbindings.yaml +++ b/config/crd/bases/camel.apache.org_kameletbindings.yaml @@ -6786,6 +6786,14 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using + `pod` strategy), sorted by the order of execution in + a csv format, ie, `<taskName1>,<taskName2>,...`. Mind + that you must include also the operator tasks (`builder`, + `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`) + if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/config/crd/bases/camel.apache.org_pipes.yaml b/config/crd/bases/camel.apache.org_pipes.yaml index 8063ab981..d018e4349 100644 --- a/config/crd/bases/camel.apache.org_pipes.yaml +++ b/config/crd/bases/camel.apache.org_pipes.yaml @@ -6784,6 +6784,14 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using + `pod` strategy), sorted by the order of execution in + a csv format, ie, `<taskName1>,<taskName2>,...`. Mind + that you must include also the operator tasks (`builder`, + `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`) + if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/docs/modules/ROOT/partials/apis/camel-k-crds.adoc b/docs/modules/ROOT/partials/apis/camel-k-crds.adoc index 4d7208467..29cc535a7 100644 --- a/docs/modules/ROOT/partials/apis/camel-k-crds.adoc +++ b/docs/modules/ROOT/partials/apis/camel-k-crds.adoc @@ -5887,6 +5887,15 @@ Syntax: [configmap{vbar}secret]:name[/key], where name represents the resource n A list of tasks to be executed (available only when using `pod` strategy) with format `<name>;<container-image>;<container-command>`. +|`tasksFilter` + +string +| + + +A list of tasks (available only when using `pod` strategy), sorted by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`. +Mind that you must include also the operator tasks (`builder`, `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`) +if you need to execute them. + |`tasksRequestCPU` + []string | diff --git a/docs/modules/traits/pages/builder.adoc b/docs/modules/traits/pages/builder.adoc index f86b3edab..6330dea1e 100755 --- a/docs/modules/traits/pages/builder.adoc +++ b/docs/modules/traits/pages/builder.adoc @@ -82,6 +82,12 @@ Syntax: [configmap\|secret]:name[/key], where name represents the resource name, | []string | A list of tasks to be executed (available only when using `pod` strategy) with format `<name>;<container-image>;<container-command>`. +| builder.tasks-filter +| string +| A list of tasks (available only when using `pod` strategy), sorted by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`. +Mind that you must include also the operator tasks (`builder`, `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`) +if you need to execute them. + | builder.tasks-request-cpu | []string | A list of request cpu configuration for the specific task with format `<task-name>:<request-cpu-conf>`. diff --git a/helm/camel-k/crds/crd-integration-kit.yaml b/helm/camel-k/crds/crd-integration-kit.yaml index e3dece12f..3d4714344 100644 --- a/helm/camel-k/crds/crd-integration-kit.yaml +++ b/helm/camel-k/crds/crd-integration-kit.yaml @@ -284,6 +284,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/helm/camel-k/crds/crd-integration-platform.yaml b/helm/camel-k/crds/crd-integration-platform.yaml index f548fe157..240fb4143 100644 --- a/helm/camel-k/crds/crd-integration-platform.yaml +++ b/helm/camel-k/crds/crd-integration-platform.yaml @@ -589,6 +589,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. @@ -2472,6 +2479,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/helm/camel-k/crds/crd-integration.yaml b/helm/camel-k/crds/crd-integration.yaml index fd9263a05..fa9fd5c0c 100644 --- a/helm/camel-k/crds/crd-integration.yaml +++ b/helm/camel-k/crds/crd-integration.yaml @@ -6502,6 +6502,13 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using `pod` + strategy), sorted by the order of execution in a csv format, + ie, `<taskName1>,<taskName2>,...`. Mind that you must include + also the operator tasks (`builder`, `quarkus-native`, `package`, + `jib`, `spectrum`, `s2i`) if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/helm/camel-k/crds/crd-kamelet-binding.yaml b/helm/camel-k/crds/crd-kamelet-binding.yaml index 0739d217f..ad82cf8c1 100644 --- a/helm/camel-k/crds/crd-kamelet-binding.yaml +++ b/helm/camel-k/crds/crd-kamelet-binding.yaml @@ -6786,6 +6786,14 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using + `pod` strategy), sorted by the order of execution in + a csv format, ie, `<taskName1>,<taskName2>,...`. Mind + that you must include also the operator tasks (`builder`, + `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`) + if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/helm/camel-k/crds/crd-pipe.yaml b/helm/camel-k/crds/crd-pipe.yaml index 8063ab981..d018e4349 100644 --- a/helm/camel-k/crds/crd-pipe.yaml +++ b/helm/camel-k/crds/crd-pipe.yaml @@ -6784,6 +6784,14 @@ spec: items: type: string type: array + tasksFilter: + description: A list of tasks (available only when using + `pod` strategy), sorted by the order of execution in + a csv format, ie, `<taskName1>,<taskName2>,...`. Mind + that you must include also the operator tasks (`builder`, + `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`) + if you need to execute them. + type: string tasksLimitCPU: description: A list of limit cpu configuration for the specific task with format `<task-name>:<limit-cpu-conf>`. diff --git a/pkg/apis/camel/v1/trait/builder.go b/pkg/apis/camel/v1/trait/builder.go index 69441859f..9dc6c361a 100644 --- a/pkg/apis/camel/v1/trait/builder.go +++ b/pkg/apis/camel/v1/trait/builder.go @@ -58,6 +58,10 @@ type BuilderTrait struct { MavenProfiles []string `property:"maven-profiles" json:"mavenProfiles,omitempty"` // A list of tasks to be executed (available only when using `pod` strategy) with format `<name>;<container-image>;<container-command>`. Tasks []string `property:"tasks" json:"tasks,omitempty"` + // A list of tasks sorted by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`. + // Mind that you must include also the operator tasks (`builder`, `quarkus-native`, `package`, `jib`, `spectrum`, `s2i`) + // if you need to execute them. Useful only wih `pod` strategy. + TasksFilter string `property:"tasks-filter" json:"tasksFilter,omitempty"` // A list of request cpu configuration for the specific task with format `<task-name>:<request-cpu-conf>`. TasksRequestCPU []string `property:"tasks-request-cpu" json:"tasksRequestCPU,omitempty"` // A list of request memory configuration for the specific task with format `<task-name>:<request-memory-conf>`. diff --git a/pkg/trait/builder.go b/pkg/trait/builder.go index fef01bcd8..87c3e484d 100644 --- a/pkg/trait/builder.go +++ b/pkg/trait/builder.go @@ -313,6 +313,13 @@ func (t *builderTrait) Apply(e *Environment) error { }}) } + // filter only those tasks required by the user + if t.TasksFilter != "" { + flt := strings.Split(t.TasksFilter, ",") + if pipelineTasks, err = filter(pipelineTasks, flt); err != nil { + return err + } + } // add local pipeline tasks to env pipeline e.Pipeline = append(e.Pipeline, pipelineTasks...) return nil @@ -559,3 +566,42 @@ func splitContainerCommand(command string) []string { return removeQuotes } + +func filter(tasks []v1.Task, filterTasks []string) ([]v1.Task, error) { + var filteredTasks []v1.Task + for _, f := range filterTasks { + found := false + for _, t := range tasks { + if t.Builder != nil && t.Builder.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } else if t.Custom != nil && t.Custom.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } else if t.Package != nil && t.Package.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } else if t.Spectrum != nil && t.Spectrum.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } else if t.S2i != nil && t.S2i.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } else if t.Jib != nil && t.Jib.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } else if t.Buildah != nil && t.Buildah.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } else if t.Kaniko != nil && t.Kaniko.Name == f { + filteredTasks = append(filteredTasks, t) + found = true + } + } + if !found { + // If we reach this point it means no tasks exists for the name + return nil, fmt.Errorf("no task exist for %s name", f) + } + } + return filteredTasks, nil +} diff --git a/pkg/trait/builder_test.go b/pkg/trait/builder_test.go index 129c4505b..343dc5e31 100644 --- a/pkg/trait/builder_test.go +++ b/pkg/trait/builder_test.go @@ -251,15 +251,6 @@ func TestCustomTaskBuilderTraitInvalidStrategyOverride(t *testing.T) { assert.Equal(t, env.IntegrationKit.Status.Conditions[0].Type, v1.IntegrationKitConditionType("IntegrationKitTasksValid")) } -func findCustomTaskByName(tasks []v1.Task, name string) v1.Task { - for _, t := range tasks { - if t.Custom != nil && t.Custom.Name == name { - return t - } - } - return v1.Task{} -} - func TestMavenProfilesBuilderTrait(t *testing.T) { env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyKaniko, v1.BuildStrategyRoutine) builderTrait := createNominalBuilderTraitTest() @@ -486,3 +477,93 @@ func TestBuilderWithNodeSelector(t *testing.T) { assert.Equal(t, map[string]string{"size": "large"}, env.Pipeline[0].Builder.Configuration.NodeSelector) assert.Equal(t, map[string]string{"size": "large"}, builderTrait.NodeSelector) } + +func TestBuilderNoTasksFilter(t *testing.T) { + env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod) + builderTrait := createNominalBuilderTraitTest() + + err := builderTrait.Apply(env) + assert.Nil(t, err) + + pipelineTasks := tasksByName(env.Pipeline) + assert.Equal(t, []string{"builder", "package", "jib"}, pipelineTasks) +} + +func TestBuilderTasksFilterNotExistingTasks(t *testing.T) { + env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod) + builderTrait := createNominalBuilderTraitTest() + builderTrait.TasksFilter = "builder,missing-task" + + err := builderTrait.Apply(env) + assert.NotNil(t, err) + assert.Equal(t, "no task exist for missing-task name", err.Error()) +} + +func TestBuilderTasksFilterOperatorTasks(t *testing.T) { + env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod) + builderTrait := createNominalBuilderTraitTest() + builderTrait.TasksFilter = "builder,package" + + err := builderTrait.Apply(env) + assert.Nil(t, err) + pipelineTasks := tasksByName(env.Pipeline) + assert.Equal(t, []string{"builder", "package"}, pipelineTasks) +} + +func TestBuilderTasksFilterAndReorderOperatorTasks(t *testing.T) { + env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod) + builderTrait := createNominalBuilderTraitTest() + builderTrait.TasksFilter = "package,builder" + + err := builderTrait.Apply(env) + assert.Nil(t, err) + pipelineTasks := tasksByName(env.Pipeline) + assert.Equal(t, []string{"package", "builder"}, pipelineTasks) +} + +func TestBuilderTasksFilterAndReorderCustomTasks(t *testing.T) { + env := createBuilderTestEnv(v1.IntegrationPlatformClusterKubernetes, v1.IntegrationPlatformBuildPublishStrategyJib, v1.BuildStrategyPod) + builderTrait := createNominalBuilderTraitTest() + builderTrait.Tasks = append(builderTrait.Tasks, `my-custom-publish;alpine;mvn test`) + builderTrait.Tasks = append(builderTrait.Tasks, "my-custom-task;alpine;ls") + builderTrait.TasksFilter = "builder,my-custom-task,package,my-custom-publish" + + err := builderTrait.Apply(env) + assert.Nil(t, err) + pipelineTasks := tasksByName(env.Pipeline) + assert.Equal(t, []string{"builder", "my-custom-task", "package", "my-custom-publish"}, pipelineTasks) +} + +func findCustomTaskByName(tasks []v1.Task, name string) v1.Task { + for _, t := range tasks { + if t.Custom != nil && t.Custom.Name == name { + return t + } + } + return v1.Task{} +} + +func tasksByName(tasks []v1.Task) []string { + pipelineTasks := make([]string, len(tasks)) + for i, t := range tasks { + if t.Builder != nil { + pipelineTasks[i] = t.Builder.Name + } + if t.Custom != nil { + pipelineTasks[i] = t.Custom.Name + } + if t.Package != nil { + pipelineTasks[i] = t.Package.Name + } + if t.S2i != nil { + pipelineTasks[i] = t.S2i.Name + } + if t.Spectrum != nil { + pipelineTasks[i] = t.Spectrum.Name + } + if t.Jib != nil { + pipelineTasks[i] = t.Jib.Name + } + } + return pipelineTasks +} diff --git a/resources/traits.yaml b/resources/traits.yaml index 0f9243967..dac366094 100755 --- a/resources/traits.yaml +++ b/resources/traits.yaml @@ -283,6 +283,12 @@ traits: type: '[]string' description: A list of tasks to be executed (available only when using `pod` strategy) with format `<name>;<container-image>;<container-command>`. + - name: tasks-filter + type: string + description: A list of tasks (available only when using `pod` strategy), sorted + by the order of execution in a csv format, ie, `<taskName1>,<taskName2>,...`. + Mind that you must include also the operator tasks (`builder`, `quarkus-native`, + `package`, `jib`, `spectrum`, `s2i`) if you need to execute them. - name: tasks-request-cpu type: '[]string' description: A list of request cpu configuration for the specific task with format
