FANNG1 commented on code in PR #7531: URL: https://github.com/apache/gravitino/pull/7531#discussion_r2182586757
########## api/src/main/java/org/apache/gravitino/job/JobHandle.java: ########## @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.job; + +/** + * JobHandle is an interface that is returned by the job submission, which provides methods to get + * the job name, job ID, job status, and to add listeners for jobs. + */ +public interface JobHandle { + + /** The status of the job. */ + enum Status { + + /** The job is in the queue and waiting to be executed. */ + QUEUED, + + /** The job is currently being executed. */ + STARTED, + + /** The job has failed during execution. */ + FAILED, + + /** The job has completed successfully. */ + SUCCEEDED, + + /** The job has been cancelled. */ + CANCELLED; + } + + /** + * Listener interface for job events. Users can implement this interface to hook in some + * customized behavior when a job is queued, started, failed, or succeeded. Note that the listener + * is running in the client side, so if the client is terminated, the listener will not work + * anymore. + */ + interface Listener { + + /** Called when the job is queued. */ + void onJobQueued(); Review Comment: Could you provide an example about how to use `Listener`? Should we provide some job context information for onJobxxx callback? ########## api/src/main/java/org/apache/gravitino/job/JobTemplate.java: ########## @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.job; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.commons.lang3.StringUtils; + +/** + * JobTemplate is a class to define all the configuration parameters for a job. + * + * <p>JobType is enum to define the type of the job, because Gravitino needs different runtime + * environments to execute different types of jobs, so the job type is required. + * + * <p>Some parameters can be templated, which means that they can be replaced with actual values + * when running the job, for example, arguments can be { "{input_path}}", "{{output_path}" }, + * environment variables can be { "foo": "{{foo_value}}", "bar": "{{bar_value}}" }. the parameters + * support templating are: + * + * <ul> + * <li>arguments + * <li>environments + * <li>configs + * </ul> + * + * The artifacts (files, jars, archives) will be uploaded to the Gravitino server and managed in the + * staging path when registering the job. + */ +public final class JobTemplate { + + /** + * JobType is an enum to define the type of the job. + * + * <p>Gravitino supports different types of jobs, such as Spark and Shell. The job type is + * required to determine the runtime environment for executing the job. + */ + public enum JobType { + + /** Job type for executing a Spark application. */ + SPARK, + + /** Job type for executing a shell command. */ + SHELL, + } + + private JobType jobType; + + private String name; + + private String comment; + + private String executable; + + private List<String> arguments; + + private Map<String, String> configs; + + private Map<String, String> environments; + + private List<String> files; Review Comment: `files`, `jars`, `archives` are the specific items for `Spark` job only, as we support more jobs, we may add more items, it's hard to maintenance. is it possible to place these items in something like `SparkSubmitContext`? ########## api/src/main/java/org/apache/gravitino/exceptions/NoSuchJobTemplateException.java: ########## @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** + * An exception thrown when the job template is not existed. This exception is typically used to + * indicate that a requested job does not exist in the system. Review Comment: requested job -> requested job template? ########## api/src/main/java/org/apache/gravitino/job/JobTemplate.java: ########## @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.job; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.commons.lang3.StringUtils; + +/** + * JobTemplate is a class to define all the configuration parameters for a job. + * + * <p>JobType is enum to define the type of the job, because Gravitino needs different runtime + * environments to execute different types of jobs, so the job type is required. + * + * <p>Some parameters can be templated, which means that they can be replaced with actual values + * when running the job, for example, arguments can be { "{input_path}}", "{{output_path}" }, + * environment variables can be { "foo": "{{foo_value}}", "bar": "{{bar_value}}" }. the parameters + * support templating are: + * + * <ul> + * <li>arguments + * <li>environments + * <li>configs + * </ul> + * + * The artifacts (files, jars, archives) will be uploaded to the Gravitino server and managed in the + * staging path when registering the job. + */ +public final class JobTemplate { Review Comment: Take the Iceberg job for example, how to pass the Iceberg jar paths to Spark command? ########## api/src/main/java/org/apache/gravitino/job/SupportsJobs.java: ########## @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.job; + +import java.util.List; +import java.util.Map; +import org.apache.gravitino.exceptions.JobTemplateAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchJobException; +import org.apache.gravitino.exceptions.NoSuchJobTemplateException; + +/** + * Interface for job management operations. This interface will be mixed with GravitinoClient to + * provide the ability to manage jobs within the Gravitino system. + */ +public interface SupportsJobs { + + /** + * Lists all the registered job templates in Gravitino. + * + * @return a list of job templates + */ + List<JobTemplate> listJobTemplates(); + + /** + * Register a job template with the specified job template to Gravitino. The registered job will + * be maintained in Gravitino, allowing it to be executed later. + * + * @param jobTemplate the template for the job + * @throws JobTemplateAlreadyExistsException if a job template with the same name already exists + */ + void registerJobTemplate(JobTemplate jobTemplate) throws JobTemplateAlreadyExistsException; + + /** + * Retrieves a job template by its name. + * + * @param jobTemplateName the name of the job template to retrieve + * @return the job template associated with the specified name + * @throws NoSuchJobTemplateException if no job template with the specified name exists + */ + JobTemplate getJobTemplate(String jobTemplateName) throws NoSuchJobTemplateException; + + /** + * Deletes a job template by its name. This will remove the job template from Gravitino, and it + * will no longer be available for execution. + * + * @param jobTemplateName the name of the job template to delete + * @throws NoSuchJobTemplateException if no job template with the specified name exists + */ + void deleteJobTemplate(String jobTemplateName) throws NoSuchJobTemplateException; + + /** + * run a job with the template name and configuration. The jobConf is a map of key-value contains + * the variables that will be used to replace the templated parameters in the job template. + * + * @param jobTemplateName the name of the job template to run + * @param jobConf the configuration for the job + * @return a handle to the run job + * @throws NoSuchJobTemplateException if no job template with the specified name exists + */ + JobHandle runJob(String jobTemplateName, Map<String, String> jobConf) Review Comment: Should we use `configs` to keep consistent with the item in `JobTempate`? ########## api/src/main/java/org/apache/gravitino/policy/Policy.java: ########## @@ -93,6 +95,15 @@ public interface Policy extends Auditable { */ Content content(); + /** + * Get the list of job templates that are associated with the policy. The job templates can be + * used to execute jobs that are related to the policy. + * + * @return The list of job templates that are associated with the policy, can be empty if no job + * templates are associated with the policy. + */ + List<JobTemplate> associatedJobTemplates(); Review Comment: Is it possible there will be multiple job template for one policy? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
