--- /dev/null
+// Copyright 2018 Google LLC.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+syntax = "proto3";
+
+package google.cloud.dataproc.v1;
+
+import "google/api/annotations.proto";
+import "google/cloud/dataproc/v1/clusters.proto";
+import "google/cloud/dataproc/v1/jobs.proto";
+import "google/longrunning/operations.proto";
+import "google/protobuf/empty.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc";
+option java_multiple_files = true;
+option java_outer_classname = "WorkflowTemplatesProto";
+option java_package = "com.google.cloud.dataproc.v1";
+
+// The API interface for managing Workflow Templates in the
+// Cloud Dataproc API.
+service WorkflowTemplateService {
+ // Creates new workflow template.
+ rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest)
+ returns (WorkflowTemplate) {
+ option (google.api.http) = {
+ post: "/v1/{parent=projects/*/locations/*}/workflowTemplates"
+ body: "template"
+ additional_bindings {
+ post: "/v1/{parent=projects/*/regions/*}/workflowTemplates"
+ body: "template"
+ }
+ };
+ }
+
+ // Retrieves the latest workflow template.
+ //
+ // Can retrieve previously instantiated template by specifying optional
+ // version parameter.
+ rpc GetWorkflowTemplate(GetWorkflowTemplateRequest)
+ returns (WorkflowTemplate) {
+ option (google.api.http) = {
+ get: "/v1/{name=projects/*/locations/*/workflowTemplates/*}"
+ additional_bindings {
+ get: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
+ }
+ };
+ }
+
+ // Instantiates a template and begins execution.
+ //
+ // The returned Operation can be used to track execution of
+ // workflow by polling
+ // [operations.get][google.longrunning.Operations.GetOperation].
+ // The Operation will complete when entire workflow is finished.
+ //
+ // The running workflow can be aborted via
+ // [operations.cancel][google.longrunning.Operations.CancelOperation].
+ // This will cause any inflight jobs to be cancelled and workflow-owned
+ // clusters to be deleted.
+ //
+ // The [Operation.metadata][google.longrunning.Operation.metadata] will be
+ // [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata].
+ //
+ // On successful completion,
+ // [Operation.response][google.longrunning.Operation.response] will be
+ // [Empty][google.protobuf.Empty].
+ rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest)
+ returns (google.longrunning.Operation) {
+ option (google.api.http) = {
+ post: "/v1/{name=projects/*/locations/*/workflowTemplates/*}:instantiate"
+ body: "*"
+ additional_bindings {
+ post: "/v1/{name=projects/*/regions/*/workflowTemplates/*}:instantiate"
+ body: "*"
+ }
+ };
+ }
+
+ // Instantiates a template and begins execution.
+ //
+ // This method is equivalent to executing the sequence
+ // [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate],
+ // [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate],
+ // [DeleteWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.DeleteWorkflowTemplate].
+ //
+ // The returned Operation can be used to track execution of
+ // workflow by polling
+ // [operations.get][google.longrunning.Operations.GetOperation].
+ // The Operation will complete when entire workflow is finished.
+ //
+ // The running workflow can be aborted via
+ // [operations.cancel][google.longrunning.Operations.CancelOperation].
+ // This will cause any inflight jobs to be cancelled and workflow-owned
+ // clusters to be deleted.
+ //
+ // The [Operation.metadata][google.longrunning.Operation.metadata] will be
+ // [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata].
+ //
+ // On successful completion,
+ // [Operation.response][google.longrunning.Operation.response] will be
+ // [Empty][google.protobuf.Empty].
+ rpc InstantiateInlineWorkflowTemplate(
+ InstantiateInlineWorkflowTemplateRequest)
+ returns (google.longrunning.Operation) {
+ option (google.api.http) = {
+ post: "/v1/{parent=projects/*/locations/*}/workflowTemplates:instantiateInline"
+ body: "template"
+ additional_bindings {
+ post: "/v1/{parent=projects/*/regions/*}/workflowTemplates:instantiateInline"
+ body: "template"
+ }
+ };
+ }
+
+ // Updates (replaces) workflow template. The updated template
+ // must contain version that matches the current server version.
+ rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest)
+ returns (WorkflowTemplate) {
+ option (google.api.http) = {
+ put: "/v1/{template.name=projects/*/locations/*/workflowTemplates/*}"
+ body: "template"
+ additional_bindings {
+ put: "/v1/{template.name=projects/*/regions/*/workflowTemplates/*}"
+ body: "template"
+ }
+ };
+ }
+
+ // Lists workflows that match the specified filter in the request.
+ rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest)
+ returns (ListWorkflowTemplatesResponse) {
+ option (google.api.http) = {
+ get: "/v1/{parent=projects/*/locations/*}/workflowTemplates"
+ additional_bindings {
+ get: "/v1/{parent=projects/*/regions/*}/workflowTemplates"
+ }
+ };
+ }
+
+ // Deletes a workflow template. It does not cancel in-progress workflows.
+ rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest)
+ returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ delete: "/v1/{name=projects/*/locations/*/workflowTemplates/*}"
+ additional_bindings {
+ delete: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
+ }
+ };
+ }
+}
+
+// A Cloud Dataproc workflow template resource.
+message WorkflowTemplate {
+ // Required. The template id.
+ //
+ // The id must contain only letters (a-z, A-Z), numbers (0-9),
+ // underscores (_), and hyphens (-). Cannot begin or end with underscore
+ // or hyphen. Must consist of between 3 and 50 characters.
+ string id = 2;
+
+ // Output only. The "resource name" of the template, as described
+ // in https://cloud.google.com/apis/design/resource_names of the form
+ // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
+ string name = 1;
+
+ // Optional. Used to perform a consistent read-modify-write.
+ //
+ // This field should be left blank for a `CreateWorkflowTemplate` request. It
+ // is required for an `UpdateWorkflowTemplate` request, and must match the
+ // current server version. A typical update template flow would fetch the
+ // current template with a `GetWorkflowTemplate` request, which will return
+ // the current template with the `version` field filled in with the
+ // current server version. The user updates other fields in the template,
+ // then returns it as part of the `UpdateWorkflowTemplate` request.
+ int32 version = 3;
+
+ // Output only. The time template was created.
+ google.protobuf.Timestamp create_time = 4;
+
+ // Output only. The time template was last updated.
+ google.protobuf.Timestamp update_time = 5;
+
+ // Optional. The labels to associate with this template. These labels
+ // will be propagated to all jobs and clusters created by the workflow
+ // instance.
+ //
+ // Label **keys** must contain 1 to 63 characters, and must conform to
+ // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
+ //
+ // Label **values** may be empty, but, if present, must contain 1 to 63
+ // characters, and must conform to
+ // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
+ //
+ // No more than 32 labels can be associated with a template.
+ map<string, string> labels = 6;
+
+ // Required. WorkflowTemplate scheduling information.
+ WorkflowTemplatePlacement placement = 7;
+
+ // Required. The Directed Acyclic Graph of Jobs to submit.
+ repeated OrderedJob jobs = 8;
+
+ // Optional. Template parameters whose values are substituted into the
+ // template. Values for parameters must be provided when the template is
+ // instantiated.
+ repeated TemplateParameter parameters = 9;
+}
+
+// Specifies workflow execution target.
+//
+// Either `managed_cluster` or `cluster_selector` is required.
+message WorkflowTemplatePlacement {
+ // Required. Specifies where workflow executes; either on a managed
+ // cluster or an existing cluster chosen by labels.
+ oneof placement {
+ // Optional. A cluster that is managed by the workflow.
+ ManagedCluster managed_cluster = 1;
+
+ // Optional. A selector that chooses target cluster for jobs based
+ // on metadata.
+ //
+ // The selector is evaluated at the time each job is submitted.
+ ClusterSelector cluster_selector = 2;
+ }
+}
+
+// Cluster that is managed by the workflow.
+message ManagedCluster {
+ // Required. The cluster name prefix. A unique cluster name will be formed by
+ // appending a random suffix.
+ //
+ // The name must contain only lower-case letters (a-z), numbers (0-9),
+ // and hyphens (-). Must begin with a letter. Cannot begin or end with
+ // hyphen. Must consist of between 2 and 35 characters.
+ string cluster_name = 2;
+
+ // Required. The cluster configuration.
+ ClusterConfig config = 3;
+
+ // Optional. The labels to associate with this cluster.
+ //
+ // Label keys must be between 1 and 63 characters long, and must conform to
+ // the following PCRE regular expression:
+ // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
+ //
+ // Label values must be between 1 and 63 characters long, and must conform to
+ // the following PCRE regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
+ //
+ // No more than 32 labels can be associated with a given cluster.
+ map<string, string> labels = 4;
+}
+
+// A selector that chooses target cluster for jobs based on metadata.
+message ClusterSelector {
+ // Optional. The zone where workflow process executes. This parameter does not
+ // affect the selection of the cluster.
+ //
+ // If unspecified, the zone of the first cluster matching the selector
+ // is used.
+ string zone = 1;
+
+ // Required. The cluster labels. Cluster must have all labels
+ // to match.
+ map<string, string> cluster_labels = 2;
+}
+
+// A job executed by the workflow.
+message OrderedJob {
+ // Required. The step id. The id must be unique among all jobs
+ // within the template.
+ //
+ // The step id is used as prefix for job id, as job
+ // `goog-dataproc-workflow-step-id` label, and in
+ // [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids]
+ // field from other steps.
+ //
+ // The id must contain only letters (a-z, A-Z), numbers (0-9),
+ // underscores (_), and hyphens (-). Cannot begin or end with underscore
+ // or hyphen. Must consist of between 3 and 50 characters.
+ string step_id = 1;
+
+ // Required. The job definition.
+ oneof job_type {
+ // Job is a Hadoop job.
+ HadoopJob hadoop_job = 2;
+
+ // Job is a Spark job.
+ SparkJob spark_job = 3;
+
+ // Job is a Pyspark job.
+ PySparkJob pyspark_job = 4;
+
+ // Job is a Hive job.
+ HiveJob hive_job = 5;
+
+ // Job is a Pig job.
+ PigJob pig_job = 6;
+
+ // Job is a SparkSql job.
+ SparkSqlJob spark_sql_job = 7;
+ }
+
+ // Optional. The labels to associate with this job.
+ //
+ // Label keys must be between 1 and 63 characters long, and must conform to
+ // the following regular expression:
+ // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
+ //
+ // Label values must be between 1 and 63 characters long, and must conform to
+ // the following regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
+ //
+ // No more than 32 labels can be associated with a given job.
+ map<string, string> labels = 8;
+
+ // Optional. Job scheduling configuration.
+ JobScheduling scheduling = 9;
+
+ // Optional. The optional list of prerequisite job step_ids.
+ // If not specified, the job will start at the beginning of workflow.
+ repeated string prerequisite_step_ids = 10;
+}
+
+// A configurable parameter that replaces one or more fields in the template.
+// Parameterizable fields:
+// - Labels
+// - File uris
+// - Job properties
+// - Job arguments
+// - Script variables
+// - Main class (in HadoopJob and SparkJob)
+// - Zone (in ClusterSelector)
+message TemplateParameter {
+ // Required. Parameter name.
+ // The parameter name is used as the key, and paired with the
+ // parameter value, which are passed to the template when the template
+ // is instantiated.
+ // The name must contain only capital letters (A-Z), numbers (0-9), and
+ // underscores (_), and must not start with a number. The maximum length is
+ // 40 characters.
+ string name = 1;
+
+ // Required. Paths to all fields that the parameter replaces.
+ // A field is allowed to appear in at most one parameter's list of field
+ // paths.
+ //
+ // A field path is similar in syntax to a
+ // [google.protobuf.FieldMask][google.protobuf.FieldMask]. For example, a
+ // field path that references the zone field of a workflow template's cluster
+ // selector would be specified as `placement.clusterSelector.zone`.
+ //
+ // Also, field paths can reference fields using the following syntax:
+ //
+ // * Values in maps can be referenced by key:
+ // * labels['key']
+ // * placement.clusterSelector.clusterLabels['key']
+ // * placement.managedCluster.labels['key']
+ // * placement.clusterSelector.clusterLabels['key']
+ // * jobs['step-id'].labels['key']
+ //
+ // * Jobs in the jobs list can be referenced by step-id:
+ // * jobs['step-id'].hadoopJob.mainJarFileUri
+ // * jobs['step-id'].hiveJob.queryFileUri
+ // * jobs['step-id'].pySparkJob.mainPythonFileUri
+ // * jobs['step-id'].hadoopJob.jarFileUris[0]
+ // * jobs['step-id'].hadoopJob.archiveUris[0]
+ // * jobs['step-id'].hadoopJob.fileUris[0]
+ // * jobs['step-id'].pySparkJob.pythonFileUris[0]
+ //
+ // * Items in repeated fields can be referenced by a zero-based index:
+ // * jobs['step-id'].sparkJob.args[0]
+ //
+ // * Other examples:
+ // * jobs['step-id'].hadoopJob.properties['key']
+ // * jobs['step-id'].hadoopJob.args[0]
+ // * jobs['step-id'].hiveJob.scriptVariables['key']
+ // * jobs['step-id'].hadoopJob.mainJarFileUri
+ // * placement.clusterSelector.zone
+ //
+ // It may not be possible to parameterize maps and repeated fields in their
+ // entirety since only individual map values and individual items in repeated
+ // fields can be referenced. For example, the following field paths are
+ // invalid:
+ //
+ // - placement.clusterSelector.clusterLabels
+ // - jobs['step-id'].sparkJob.args
+ repeated string fields = 2;
+
+ // Optional. Brief description of the parameter.
+ // Must not exceed 1024 characters.
+ string description = 3;
+
+ // Optional. Validation rules to be applied to this parameter's value.
+ ParameterValidation validation = 4;
+}
+
+// Configuration for parameter validation.
+message ParameterValidation {
+ // Required. The type of validation to be performed.
+ oneof validation_type {
+ // Validation based on regular expressions.
+ RegexValidation regex = 1;
+
+ // Validation based on a list of allowed values.
+ ValueValidation values = 2;
+ }
+}
+
+// Validation based on regular expressions.
+message RegexValidation {
+ // Required. RE2 regular expressions used to validate the parameter's value.
+ // The value must match the regex in its entirety (substring
+ // matches are not sufficient).
+ repeated string regexes = 1;
+}
+
+// Validation based on a list of allowed values.
+message ValueValidation {
+ // Required. List of allowed values for the parameter.
+ repeated string values = 1;
+}
+
+// A Cloud Dataproc workflow template resource.
+message WorkflowMetadata {
+ // The operation state.
+ enum State {
+ // Unused.
+ UNKNOWN = 0;
+
+ // The operation has been created.
+ PENDING = 1;
+
+ // The operation is running.
+ RUNNING = 2;
+
+ // The operation is done; either cancelled or completed.
+ DONE = 3;
+ }
+
+ // Output only. The "resource name" of the template.
+ string template = 1;
+
+ // Output only. The version of template at the time of
+ // workflow instantiation.
+ int32 version = 2;
+
+ // Output only. The create cluster operation metadata.
+ ClusterOperation create_cluster = 3;
+
+ // Output only. The workflow graph.
+ WorkflowGraph graph = 4;
+
+ // Output only. The delete cluster operation metadata.
+ ClusterOperation delete_cluster = 5;
+
+ // Output only. The workflow state.
+ State state = 6;
+
+ // Output only. The name of the target cluster.
+ string cluster_name = 7;
+
+ // Map from parameter names to values that were used for those parameters.
+ map<string, string> parameters = 8;
+
+ // Output only. Workflow start time.
+ google.protobuf.Timestamp start_time = 9;
+
+ // Output only. Workflow end time.
+ google.protobuf.Timestamp end_time = 10;
+
+ // Output only. The UUID of target cluster.
+ string cluster_uuid = 11;
+}
+
+// The cluster operation triggered by a workflow.
+message ClusterOperation {
+ // Output only. The id of the cluster operation.
+ string operation_id = 1;
+
+ // Output only. Error, if operation failed.
+ string error = 2;
+
+ // Output only. Indicates the operation is done.
+ bool done = 3;
+}
+
+// The workflow graph.
+message WorkflowGraph {
+ // Output only. The workflow nodes.
+ repeated WorkflowNode nodes = 1;
+}
+
+// The workflow node.
+message WorkflowNode {
+ // The workflow node state.
+ enum NodeState {
+ // State is unspecified.
+ NODE_STATE_UNSPECIFIED = 0;
+
+ // The node is awaiting prerequisite node to finish.
+ BLOCKED = 1;
+
+ // The node is runnable but not running.
+ RUNNABLE = 2;
+
+ // The node is running.
+ RUNNING = 3;
+
+ // The node completed successfully.
+ COMPLETED = 4;
+
+ // The node failed. A node can be marked FAILED because
+ // its ancestor or peer failed.
+ FAILED = 5;
+ }
+
+ // Output only. The name of the node.
+ string step_id = 1;
+
+ // Output only. Node's prerequisite nodes.
+ repeated string prerequisite_step_ids = 2;
+
+ // Output only. The job id; populated after the node enters RUNNING state.
+ string job_id = 3;
+
+ // Output only. The node state.
+ NodeState state = 5;
+
+ // Output only. The error detail.
+ string error = 6;
+}
+
+// A request to create a workflow template.
+message CreateWorkflowTemplateRequest {
+ // Required. The "resource name" of the region, as described
+ // in https://cloud.google.com/apis/design/resource_names of the form
+ // `projects/{project_id}/regions/{region}`
+ string parent = 1;
+
+ // Required. The Dataproc workflow template to create.
+ WorkflowTemplate template = 2;
+}
+
+// A request to fetch a workflow template.
+message GetWorkflowTemplateRequest {
+ // Required. The "resource name" of the workflow template, as described
+ // in https://cloud.google.com/apis/design/resource_names of the form
+ // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
+ string name = 1;
+
+ // Optional. The version of workflow template to retrieve. Only previously
+ // instatiated versions can be retrieved.
+ //
+ // If unspecified, retrieves the current version.
+ int32 version = 2;
+}
+
+// A request to instantiate a workflow template.
+message InstantiateWorkflowTemplateRequest {
+ // Required. The "resource name" of the workflow template, as described
+ // in https://cloud.google.com/apis/design/resource_names of the form
+ // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
+ string name = 1;
+
+ // Optional. The version of workflow template to instantiate. If specified,
+ // the workflow will be instantiated only if the current version of
+ // the workflow template has the supplied version.
+ //
+ // This option cannot be used to instantiate a previous version of
+ // workflow template.
+ int32 version = 2;
+
+ // Optional. A tag that prevents multiple concurrent workflow
+ // instances with the same tag from running. This mitigates risk of
+ // concurrent instances started due to retries.
+ //
+ // It is recommended to always set this value to a
+ // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
+ //
+ // The tag must contain only letters (a-z, A-Z), numbers (0-9),
+ // underscores (_), and hyphens (-). The maximum length is 40 characters.
+ string request_id = 5;
+
+ // Optional. Map from parameter names to values that should be used for those
+ // parameters. Values may not exceed 100 characters.
+ map<string, string> parameters = 6;
+}
+
+// A request to instantiate an inline workflow template.
+message InstantiateInlineWorkflowTemplateRequest {
+ // Required. The "resource name" of the workflow template region, as described
+ // in https://cloud.google.com/apis/design/resource_names of the form
+ // `projects/{project_id}/regions/{region}`
+ string parent = 1;
+
+ // Required. The workflow template to instantiate.
+ WorkflowTemplate template = 2;
+
+ // Optional. A tag that prevents multiple concurrent workflow
+ // instances with the same tag from running. This mitigates risk of
+ // concurrent instances started due to retries.
+ //
+ // It is recommended to always set this value to a
+ // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
+ //
+ // The tag must contain only letters (a-z, A-Z), numbers (0-9),
+ // underscores (_), and hyphens (-). The maximum length is 40 characters.
+ string request_id = 3;
+}
+
+// A request to update a workflow template.
+message UpdateWorkflowTemplateRequest {
+ // Required. The updated workflow template.
+ //
+ // The `template.version` field must match the current version.
+ WorkflowTemplate template = 1;
+}
+
+// A request to list workflow templates in a project.
+message ListWorkflowTemplatesRequest {
+ // Required. The "resource name" of the region, as described
+ // in https://cloud.google.com/apis/design/resource_names of the form
+ // `projects/{project_id}/regions/{region}`
+ string parent = 1;
+
+ // Optional. The maximum number of results to return in each response.
+ int32 page_size = 2;
+
+ // Optional. The page token, returned by a previous call, to request the
+ // next page of results.
+ string page_token = 3;
+}
+
+// A response to a request to list workflow templates in a project.
+message ListWorkflowTemplatesResponse {
+ // Output only. WorkflowTemplates list.
+ repeated WorkflowTemplate templates = 1;
+
+ // Output only. This token is included in the response if there are more
+ // results to fetch. To fetch additional results, provide this value as the
+ // page_token in a subsequent <code>ListWorkflowTemplatesRequest</code>.
+ string next_page_token = 2;
+}
+
+// A request to delete a workflow template.
+//
+// Currently started workflows will remain running.
+message DeleteWorkflowTemplateRequest {
+ // Required. The "resource name" of the workflow template, as described
+ // in https://cloud.google.com/apis/design/resource_names of the form
+ // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
+ string name = 1;
+
+ // Optional. The version of workflow template to delete. If specified,
+ // will only delete the template if the current server version matches
+ // specified version.
+ int32 version = 2;
+}
git://repos.xcallymotion.com / motion2.git / blobdiff