Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(207)

Issues Search
    My Issues | Starred     Open | Closed | All

Side by Side Diff: discovery/googleapis_beta/dataflow__v1b3.json

Issue 2695743002: Api-roll 45: 2017-02-13 (Closed)
Patch Set: reverted local changes to pubspec file Created 3 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « discovery/googleapis_beta/cloudresourcemanager__v1beta1.json ('k') | discovery/googleapis_beta/dataproc__v1beta1.json » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 { 1 {
2 "auth": { 2 "auth": {
3 "oauth2": { 3 "oauth2": {
4 "scopes": { 4 "scopes": {
5 "https://www.googleapis.com/auth/cloud-platform": { 5 "https://www.googleapis.com/auth/cloud-platform": {
6 "description": "View and manage your data across Google Clou d Platform services" 6 "description": "View and manage your data across Google Clou d Platform services"
7 }, 7 },
8 "https://www.googleapis.com/auth/userinfo.email": { 8 "https://www.googleapis.com/auth/userinfo.email": {
9 "description": "View your email address" 9 "description": "View your email address"
10 } 10 }
11 } 11 }
12 } 12 }
13 }, 13 },
14 "basePath": "", 14 "basePath": "",
15 "baseUrl": "https://dataflow.googleapis.com/", 15 "baseUrl": "https://dataflow.googleapis.com/",
16 "batchPath": "batch", 16 "batchPath": "batch",
17 "description": "Manages Google Cloud Dataflow projects on Google Cloud Platf orm.", 17 "description": "Manages Google Cloud Dataflow projects on Google Cloud Platf orm.",
18 "discoveryVersion": "v1", 18 "discoveryVersion": "v1",
19 "documentationLink": "https://cloud.google.com/dataflow", 19 "documentationLink": "https://cloud.google.com/dataflow",
20 "etag": "\"tbys6C40o18GZwyMen5GMkdK-3s/8IKdD_GkxmHITZPPBvAy8YZADEQ\"",
21 "icons": { 20 "icons": {
22 "x16": "http://www.google.com/images/icons/product/search-16.gif", 21 "x16": "http://www.google.com/images/icons/product/search-16.gif",
23 "x32": "http://www.google.com/images/icons/product/search-32.gif" 22 "x32": "http://www.google.com/images/icons/product/search-32.gif"
24 }, 23 },
25 "id": "dataflow:v1b3", 24 "id": "dataflow:v1b3",
26 "kind": "discovery#restDescription", 25 "kind": "discovery#restDescription",
27 "name": "dataflow", 26 "name": "dataflow",
28 "ownerDomain": "google.com", 27 "ownerDomain": "google.com",
29 "ownerName": "Google", 28 "ownerName": "Google",
30 "parameters": { 29 "parameters": {
31 "access_token": { 30 "quotaUser": {
32 "description": "OAuth access token.", 31 "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exc eed 40 characters.",
33 "location": "query",
34 "type": "string"
35 },
36 "alt": {
37 "default": "json",
38 "description": "Data format for response.",
39 "enumDescriptions": [
40 "Responses with Content-Type of application/json",
41 "Media download with context-dependent Content-Type",
42 "Responses with Content-Type of application/x-protobuf"
43 ],
44 "location": "query",
45 "type": "string"
46 },
47 "bearer_token": {
48 "description": "OAuth bearer token.",
49 "location": "query",
50 "type": "string"
51 },
52 "callback": {
53 "description": "JSONP",
54 "location": "query",
55 "type": "string"
56 },
57 "fields": {
58 "description": "Selector specifying which fields to include in a par tial response.",
59 "location": "query",
60 "type": "string"
61 },
62 "key": {
63 "description": "API key. Your API key identifies your project and pr ovides you with API access, quota, and reports. Required unless you provide an O Auth 2.0 token.",
64 "location": "query",
65 "type": "string"
66 },
67 "oauth_token": {
68 "description": "OAuth 2.0 token for the current user.",
69 "location": "query", 32 "location": "query",
70 "type": "string" 33 "type": "string"
71 }, 34 },
72 "pp": { 35 "pp": {
73 "default": "true", 36 "default": "true",
74 "description": "Pretty-print response.", 37 "description": "Pretty-print response.",
75 "location": "query", 38 "location": "query",
76 "type": "boolean" 39 "type": "boolean"
77 }, 40 },
41 "bearer_token": {
42 "description": "OAuth bearer token.",
43 "location": "query",
44 "type": "string"
45 },
46 "oauth_token": {
47 "description": "OAuth 2.0 token for the current user.",
48 "location": "query",
49 "type": "string"
50 },
51 "upload_protocol": {
52 "description": "Upload protocol for media (e.g. \"raw\", \"multipart \").",
53 "location": "query",
54 "type": "string"
55 },
78 "prettyPrint": { 56 "prettyPrint": {
79 "default": "true", 57 "default": "true",
80 "description": "Returns response with indentations and line breaks." , 58 "description": "Returns response with indentations and line breaks." ,
81 "location": "query", 59 "location": "query",
82 "type": "boolean" 60 "type": "boolean"
83 }, 61 },
84 "quotaUser": {
85 "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exc eed 40 characters.",
86 "location": "query",
87 "type": "string"
88 },
89 "upload_protocol": {
90 "description": "Upload protocol for media (e.g. \"raw\", \"multipart \").",
91 "location": "query",
92 "type": "string"
93 },
94 "uploadType": { 62 "uploadType": {
95 "description": "Legacy upload protocol for media (e.g. \"media\", \" multipart\").", 63 "description": "Legacy upload protocol for media (e.g. \"media\", \" multipart\").",
96 "location": "query", 64 "location": "query",
97 "type": "string" 65 "type": "string"
98 }, 66 },
67 "fields": {
68 "description": "Selector specifying which fields to include in a par tial response.",
69 "location": "query",
70 "type": "string"
71 },
72 "callback": {
73 "description": "JSONP",
74 "location": "query",
75 "type": "string"
76 },
99 "$.xgafv": { 77 "$.xgafv": {
100 "description": "V1 error format.", 78 "description": "V1 error format.",
79 "enum": [
80 "1",
81 "2"
82 ],
101 "enumDescriptions": [ 83 "enumDescriptions": [
102 "v1 error format", 84 "v1 error format",
103 "v2 error format" 85 "v2 error format"
104 ], 86 ],
105 "location": "query", 87 "location": "query",
106 "type": "string" 88 "type": "string"
89 },
90 "alt": {
91 "default": "json",
92 "description": "Data format for response.",
93 "enum": [
94 "json",
95 "media",
96 "proto"
97 ],
98 "enumDescriptions": [
99 "Responses with Content-Type of application/json",
100 "Media download with context-dependent Content-Type",
101 "Responses with Content-Type of application/x-protobuf"
102 ],
103 "location": "query",
104 "type": "string"
105 },
106 "key": {
107 "description": "API key. Your API key identifies your project and pr ovides you with API access, quota, and reports. Required unless you provide an O Auth 2.0 token.",
108 "location": "query",
109 "type": "string"
110 },
111 "access_token": {
112 "description": "OAuth access token.",
113 "location": "query",
114 "type": "string"
107 } 115 }
108 }, 116 },
109 "protocol": "rest", 117 "protocol": "rest",
110 "resources": { 118 "resources": {
111 "projects": { 119 "projects": {
112 "methods": { 120 "methods": {
113 "workerMessages": { 121 "workerMessages": {
114 "description": "Send a worker_message to the service.", 122 "description": "Send a worker_message to the service.",
115 "httpMethod": "POST", 123 "httpMethod": "POST",
116 "id": "dataflow.projects.workerMessages", 124 "id": "dataflow.projects.workerMessages",
(...skipping 15 matching lines...) Expand all
132 "response": { 140 "response": {
133 "$ref": "SendWorkerMessagesResponse" 141 "$ref": "SendWorkerMessagesResponse"
134 }, 142 },
135 "scopes": [ 143 "scopes": [
136 "https://www.googleapis.com/auth/cloud-platform", 144 "https://www.googleapis.com/auth/cloud-platform",
137 "https://www.googleapis.com/auth/userinfo.email" 145 "https://www.googleapis.com/auth/userinfo.email"
138 ] 146 ]
139 } 147 }
140 }, 148 },
141 "resources": { 149 "resources": {
142 "jobs": {
143 "methods": {
144 "create": {
145 "description": "Creates a Cloud Dataflow job.",
146 "httpMethod": "POST",
147 "id": "dataflow.projects.jobs.create",
148 "parameterOrder": [
149 "projectId"
150 ],
151 "parameters": {
152 "projectId": {
153 "description": "The ID of the Cloud Platform project that the job belongs to.",
154 "location": "path",
155 "required": true,
156 "type": "string"
157 },
158 "view": {
159 "description": "The level of information req uested in response.",
160 "enum": [
161 "JOB_VIEW_UNKNOWN",
162 "JOB_VIEW_SUMMARY",
163 "JOB_VIEW_ALL"
164 ],
165 "location": "query",
166 "type": "string"
167 },
168 "replaceJobId": {
169 "description": "Deprecated. This field is no w in the Job message.",
170 "location": "query",
171 "type": "string"
172 },
173 "location": {
174 "description": "The location that contains t his job.",
175 "location": "query",
176 "type": "string"
177 }
178 },
179 "path": "v1b3/projects/{projectId}/jobs",
180 "request": {
181 "$ref": "Job"
182 },
183 "response": {
184 "$ref": "Job"
185 },
186 "scopes": [
187 "https://www.googleapis.com/auth/cloud-platform" ,
188 "https://www.googleapis.com/auth/userinfo.email"
189 ]
190 },
191 "get": {
192 "description": "Gets the state of the specified Clou d Dataflow job.",
193 "httpMethod": "GET",
194 "id": "dataflow.projects.jobs.get",
195 "parameterOrder": [
196 "projectId",
197 "jobId"
198 ],
199 "parameters": {
200 "projectId": {
201 "description": "The ID of the Cloud Platform project that the job belongs to.",
202 "location": "path",
203 "required": true,
204 "type": "string"
205 },
206 "jobId": {
207 "description": "The job ID.",
208 "location": "path",
209 "required": true,
210 "type": "string"
211 },
212 "view": {
213 "description": "The level of information req uested in response.",
214 "enum": [
215 "JOB_VIEW_UNKNOWN",
216 "JOB_VIEW_SUMMARY",
217 "JOB_VIEW_ALL"
218 ],
219 "location": "query",
220 "type": "string"
221 },
222 "location": {
223 "description": "The location that contains t his job.",
224 "location": "query",
225 "type": "string"
226 }
227 },
228 "path": "v1b3/projects/{projectId}/jobs/{jobId}",
229 "response": {
230 "$ref": "Job"
231 },
232 "scopes": [
233 "https://www.googleapis.com/auth/cloud-platform" ,
234 "https://www.googleapis.com/auth/userinfo.email"
235 ]
236 },
237 "update": {
238 "description": "Updates the state of an existing Clo ud Dataflow job.",
239 "httpMethod": "PUT",
240 "id": "dataflow.projects.jobs.update",
241 "parameterOrder": [
242 "projectId",
243 "jobId"
244 ],
245 "parameters": {
246 "projectId": {
247 "description": "The ID of the Cloud Platform project that the job belongs to.",
248 "location": "path",
249 "required": true,
250 "type": "string"
251 },
252 "jobId": {
253 "description": "The job ID.",
254 "location": "path",
255 "required": true,
256 "type": "string"
257 },
258 "location": {
259 "description": "The location that contains t his job.",
260 "location": "query",
261 "type": "string"
262 }
263 },
264 "path": "v1b3/projects/{projectId}/jobs/{jobId}",
265 "request": {
266 "$ref": "Job"
267 },
268 "response": {
269 "$ref": "Job"
270 },
271 "scopes": [
272 "https://www.googleapis.com/auth/cloud-platform" ,
273 "https://www.googleapis.com/auth/userinfo.email"
274 ]
275 },
276 "list": {
277 "description": "List the jobs of a project.",
278 "httpMethod": "GET",
279 "id": "dataflow.projects.jobs.list",
280 "parameterOrder": [
281 "projectId"
282 ],
283 "parameters": {
284 "projectId": {
285 "description": "The project which owns the j obs.",
286 "location": "path",
287 "required": true,
288 "type": "string"
289 },
290 "filter": {
291 "description": "The kind of filter to use.",
292 "enum": [
293 "UNKNOWN",
294 "ALL",
295 "TERMINATED",
296 "ACTIVE"
297 ],
298 "location": "query",
299 "type": "string"
300 },
301 "view": {
302 "description": "Level of information request ed in response. Default is `JOB_VIEW_SUMMARY`.",
303 "enum": [
304 "JOB_VIEW_UNKNOWN",
305 "JOB_VIEW_SUMMARY",
306 "JOB_VIEW_ALL"
307 ],
308 "location": "query",
309 "type": "string"
310 },
311 "pageSize": {
312 "description": "If there are many jobs, limi t response to at most this many. The actual number of jobs returned will be the lesser of max_responses and an unspecified server-defined limit.",
313 "format": "int32",
314 "location": "query",
315 "type": "integer"
316 },
317 "pageToken": {
318 "description": "Set this to the 'next_page_t oken' field of a previous response to request additional results in a long list. ",
319 "location": "query",
320 "type": "string"
321 },
322 "location": {
323 "description": "The location that contains t his job.",
324 "location": "query",
325 "type": "string"
326 }
327 },
328 "path": "v1b3/projects/{projectId}/jobs",
329 "response": {
330 "$ref": "ListJobsResponse"
331 },
332 "scopes": [
333 "https://www.googleapis.com/auth/cloud-platform" ,
334 "https://www.googleapis.com/auth/userinfo.email"
335 ]
336 },
337 "getMetrics": {
338 "description": "Request the job status.",
339 "httpMethod": "GET",
340 "id": "dataflow.projects.jobs.getMetrics",
341 "parameterOrder": [
342 "projectId",
343 "jobId"
344 ],
345 "parameters": {
346 "projectId": {
347 "description": "A project id.",
348 "location": "path",
349 "required": true,
350 "type": "string"
351 },
352 "jobId": {
353 "description": "The job to get messages for. ",
354 "location": "path",
355 "required": true,
356 "type": "string"
357 },
358 "startTime": {
359 "description": "Return only metric data that has changed since this time. Default is to return all information about all met rics for the job.",
360 "location": "query",
361 "type": "string"
362 },
363 "location": {
364 "description": "The location which contains the job specified by job_id.",
365 "location": "query",
366 "type": "string"
367 }
368 },
369 "path": "v1b3/projects/{projectId}/jobs/{jobId}/metr ics",
370 "response": {
371 "$ref": "JobMetrics"
372 },
373 "scopes": [
374 "https://www.googleapis.com/auth/cloud-platform" ,
375 "https://www.googleapis.com/auth/userinfo.email"
376 ]
377 }
378 },
379 "resources": {
380 "debug": {
381 "methods": {
382 "getConfig": {
383 "description": "Get encoded debug configurat ion for component. Not cacheable.",
384 "httpMethod": "POST",
385 "id": "dataflow.projects.jobs.debug.getConfi g",
386 "parameterOrder": [
387 "projectId",
388 "jobId"
389 ],
390 "parameters": {
391 "projectId": {
392 "description": "The project id.",
393 "location": "path",
394 "required": true,
395 "type": "string"
396 },
397 "jobId": {
398 "description": "The job id.",
399 "location": "path",
400 "required": true,
401 "type": "string"
402 }
403 },
404 "path": "v1b3/projects/{projectId}/jobs/{job Id}/debug/getConfig",
405 "request": {
406 "$ref": "GetDebugConfigRequest"
407 },
408 "response": {
409 "$ref": "GetDebugConfigResponse"
410 },
411 "scopes": [
412 "https://www.googleapis.com/auth/cloud-p latform",
413 "https://www.googleapis.com/auth/userinf o.email"
414 ]
415 },
416 "sendCapture": {
417 "description": "Send encoded debug capture d ata for component.",
418 "httpMethod": "POST",
419 "id": "dataflow.projects.jobs.debug.sendCapt ure",
420 "parameterOrder": [
421 "projectId",
422 "jobId"
423 ],
424 "parameters": {
425 "projectId": {
426 "description": "The project id.",
427 "location": "path",
428 "required": true,
429 "type": "string"
430 },
431 "jobId": {
432 "description": "The job id.",
433 "location": "path",
434 "required": true,
435 "type": "string"
436 }
437 },
438 "path": "v1b3/projects/{projectId}/jobs/{job Id}/debug/sendCapture",
439 "request": {
440 "$ref": "SendDebugCaptureRequest"
441 },
442 "response": {
443 "$ref": "SendDebugCaptureResponse"
444 },
445 "scopes": [
446 "https://www.googleapis.com/auth/cloud-p latform",
447 "https://www.googleapis.com/auth/userinf o.email"
448 ]
449 }
450 }
451 },
452 "messages": {
453 "methods": {
454 "list": {
455 "description": "Request the job status.",
456 "httpMethod": "GET",
457 "id": "dataflow.projects.jobs.messages.list" ,
458 "parameterOrder": [
459 "projectId",
460 "jobId"
461 ],
462 "parameters": {
463 "projectId": {
464 "description": "A project id.",
465 "location": "path",
466 "required": true,
467 "type": "string"
468 },
469 "jobId": {
470 "description": "The job to get messa ges about.",
471 "location": "path",
472 "required": true,
473 "type": "string"
474 },
475 "minimumImportance": {
476 "description": "Filter to only get m essages with importance >= level",
477 "enum": [
478 "JOB_MESSAGE_IMPORTANCE_UNKNOWN" ,
479 "JOB_MESSAGE_DEBUG",
480 "JOB_MESSAGE_DETAILED",
481 "JOB_MESSAGE_BASIC",
482 "JOB_MESSAGE_WARNING",
483 "JOB_MESSAGE_ERROR"
484 ],
485 "location": "query",
486 "type": "string"
487 },
488 "pageSize": {
489 "description": "If specified, determ ines the maximum number of messages to return. If unspecified, the service may c hoose an appropriate default, or may return an arbitrarily large number of resul ts.",
490 "format": "int32",
491 "location": "query",
492 "type": "integer"
493 },
494 "pageToken": {
495 "description": "If supplied, this sh ould be the value of next_page_token returned by an earlier call. This will caus e the next page of results to be returned.",
496 "location": "query",
497 "type": "string"
498 },
499 "startTime": {
500 "description": "If specified, return only messages with timestamps >= start_time. The default is the job creation ti me (i.e. beginning of messages).",
501 "location": "query",
502 "type": "string"
503 },
504 "endTime": {
505 "description": "Return only messages with timestamps < end_time. The default is now (i.e. return up to the latest me ssages available).",
506 "location": "query",
507 "type": "string"
508 },
509 "location": {
510 "description": "The location which c ontains the job specified by job_id.",
511 "location": "query",
512 "type": "string"
513 }
514 },
515 "path": "v1b3/projects/{projectId}/jobs/{job Id}/messages",
516 "response": {
517 "$ref": "ListJobMessagesResponse"
518 },
519 "scopes": [
520 "https://www.googleapis.com/auth/cloud-p latform",
521 "https://www.googleapis.com/auth/userinf o.email"
522 ]
523 }
524 }
525 },
526 "workItems": {
527 "methods": {
528 "reportStatus": {
529 "description": "Reports the status of datafl ow WorkItems leased by a worker.",
530 "httpMethod": "POST",
531 "id": "dataflow.projects.jobs.workItems.repo rtStatus",
532 "parameterOrder": [
533 "projectId",
534 "jobId"
535 ],
536 "parameters": {
537 "projectId": {
538 "description": "The project which ow ns the WorkItem's job.",
539 "location": "path",
540 "required": true,
541 "type": "string"
542 },
543 "jobId": {
544 "description": "The job which the Wo rkItem is part of.",
545 "location": "path",
546 "required": true,
547 "type": "string"
548 }
549 },
550 "path": "v1b3/projects/{projectId}/jobs/{job Id}/workItems:reportStatus",
551 "request": {
552 "$ref": "ReportWorkItemStatusRequest"
553 },
554 "response": {
555 "$ref": "ReportWorkItemStatusResponse"
556 },
557 "scopes": [
558 "https://www.googleapis.com/auth/cloud-p latform",
559 "https://www.googleapis.com/auth/userinf o.email"
560 ]
561 },
562 "lease": {
563 "description": "Leases a dataflow WorkItem t o run.",
564 "httpMethod": "POST",
565 "id": "dataflow.projects.jobs.workItems.leas e",
566 "parameterOrder": [
567 "projectId",
568 "jobId"
569 ],
570 "parameters": {
571 "projectId": {
572 "description": "Identifies the proje ct this worker belongs to.",
573 "location": "path",
574 "required": true,
575 "type": "string"
576 },
577 "jobId": {
578 "description": "Identifies the workf low job this worker belongs to.",
579 "location": "path",
580 "required": true,
581 "type": "string"
582 }
583 },
584 "path": "v1b3/projects/{projectId}/jobs/{job Id}/workItems:lease",
585 "request": {
586 "$ref": "LeaseWorkItemRequest"
587 },
588 "response": {
589 "$ref": "LeaseWorkItemResponse"
590 },
591 "scopes": [
592 "https://www.googleapis.com/auth/cloud-p latform",
593 "https://www.googleapis.com/auth/userinf o.email"
594 ]
595 }
596 }
597 }
598 }
599 },
600 "locations": { 150 "locations": {
601 "resources": { 151 "resources": {
602 "jobs": { 152 "jobs": {
603 "methods": { 153 "methods": {
154 "getMetrics": {
155 "description": "Request the job status.",
156 "httpMethod": "GET",
157 "id": "dataflow.projects.locations.jobs.getM etrics",
158 "parameterOrder": [
159 "projectId",
160 "location",
161 "jobId"
162 ],
163 "parameters": {
164 "location": {
165 "description": "The location which c ontains the job specified by job_id.",
166 "location": "path",
167 "required": true,
168 "type": "string"
169 },
170 "startTime": {
171 "description": "Return only metric d ata that has changed since this time.\nDefault is to return all information abou t all metrics for the job.",
172 "format": "google-datetime",
173 "location": "query",
174 "type": "string"
175 },
176 "projectId": {
177 "description": "A project id.",
178 "location": "path",
179 "required": true,
180 "type": "string"
181 },
182 "jobId": {
183 "description": "The job to get messa ges for.",
184 "location": "path",
185 "required": true,
186 "type": "string"
187 }
188 },
189 "path": "v1b3/projects/{projectId}/locations /{location}/jobs/{jobId}/metrics",
190 "response": {
191 "$ref": "JobMetrics"
192 },
193 "scopes": [
194 "https://www.googleapis.com/auth/cloud-p latform",
195 "https://www.googleapis.com/auth/userinf o.email"
196 ]
197 },
198 "get": {
199 "description": "Gets the state of the specif ied Cloud Dataflow job.",
200 "httpMethod": "GET",
201 "id": "dataflow.projects.locations.jobs.get" ,
202 "parameterOrder": [
203 "projectId",
204 "location",
205 "jobId"
206 ],
207 "parameters": {
208 "location": {
209 "description": "The location that co ntains this job.",
210 "location": "path",
211 "required": true,
212 "type": "string"
213 },
214 "projectId": {
215 "description": "The ID of the Cloud Platform project that the job belongs to.",
216 "location": "path",
217 "required": true,
218 "type": "string"
219 },
220 "jobId": {
221 "description": "The job ID.",
222 "location": "path",
223 "required": true,
224 "type": "string"
225 },
226 "view": {
227 "description": "The level of informa tion requested in response.",
228 "enum": [
229 "JOB_VIEW_UNKNOWN",
230 "JOB_VIEW_SUMMARY",
231 "JOB_VIEW_ALL"
232 ],
233 "location": "query",
234 "type": "string"
235 }
236 },
237 "path": "v1b3/projects/{projectId}/locations /{location}/jobs/{jobId}",
238 "response": {
239 "$ref": "Job"
240 },
241 "scopes": [
242 "https://www.googleapis.com/auth/cloud-p latform",
243 "https://www.googleapis.com/auth/userinf o.email"
244 ]
245 },
246 "list": {
247 "description": "List the jobs of a project." ,
248 "httpMethod": "GET",
249 "id": "dataflow.projects.locations.jobs.list ",
250 "parameterOrder": [
251 "projectId",
252 "location"
253 ],
254 "parameters": {
255 "projectId": {
256 "description": "The project which ow ns the jobs.",
257 "location": "path",
258 "required": true,
259 "type": "string"
260 },
261 "filter": {
262 "description": "The kind of filter t o use.",
263 "enum": [
264 "UNKNOWN",
265 "ALL",
266 "TERMINATED",
267 "ACTIVE"
268 ],
269 "location": "query",
270 "type": "string"
271 },
272 "location": {
273 "description": "The location that co ntains this job.",
274 "location": "path",
275 "required": true,
276 "type": "string"
277 },
278 "pageToken": {
279 "description": "Set this to the 'nex t_page_token' field of a previous response\nto request additional results in a l ong list.",
280 "location": "query",
281 "type": "string"
282 },
283 "pageSize": {
284 "description": "If there are many jo bs, limit response to at most this many.\nThe actual number of jobs returned wil l be the lesser of max_responses\nand an unspecified server-defined limit.",
285 "format": "int32",
286 "location": "query",
287 "type": "integer"
288 },
289 "view": {
290 "description": "Level of information requested in response. Default is `JOB_VIEW_SUMMARY`.",
291 "enum": [
292 "JOB_VIEW_UNKNOWN",
293 "JOB_VIEW_SUMMARY",
294 "JOB_VIEW_ALL"
295 ],
296 "location": "query",
297 "type": "string"
298 }
299 },
300 "path": "v1b3/projects/{projectId}/locations /{location}/jobs",
301 "response": {
302 "$ref": "ListJobsResponse"
303 },
304 "scopes": [
305 "https://www.googleapis.com/auth/cloud-p latform",
306 "https://www.googleapis.com/auth/userinf o.email"
307 ]
308 },
309 "update": {
310 "description": "Updates the state of an exis ting Cloud Dataflow job.",
311 "httpMethod": "PUT",
312 "id": "dataflow.projects.locations.jobs.upda te",
313 "parameterOrder": [
314 "projectId",
315 "location",
316 "jobId"
317 ],
318 "parameters": {
319 "location": {
320 "description": "The location that co ntains this job.",
321 "location": "path",
322 "required": true,
323 "type": "string"
324 },
325 "projectId": {
326 "description": "The ID of the Cloud Platform project that the job belongs to.",
327 "location": "path",
328 "required": true,
329 "type": "string"
330 },
331 "jobId": {
332 "description": "The job ID.",
333 "location": "path",
334 "required": true,
335 "type": "string"
336 }
337 },
338 "path": "v1b3/projects/{projectId}/locations /{location}/jobs/{jobId}",
339 "request": {
340 "$ref": "Job"
341 },
342 "response": {
343 "$ref": "Job"
344 },
345 "scopes": [
346 "https://www.googleapis.com/auth/cloud-p latform",
347 "https://www.googleapis.com/auth/userinf o.email"
348 ]
349 },
604 "create": { 350 "create": {
605 "description": "Creates a Cloud Dataflow job .", 351 "description": "Creates a Cloud Dataflow job .",
606 "httpMethod": "POST", 352 "httpMethod": "POST",
607 "id": "dataflow.projects.locations.jobs.crea te", 353 "id": "dataflow.projects.locations.jobs.crea te",
608 "parameterOrder": [ 354 "parameterOrder": [
609 "projectId", 355 "projectId",
610 "location" 356 "location"
611 ], 357 ],
612 "parameters": { 358 "parameters": {
613 "projectId": { 359 "projectId": {
614 "description": "The ID of the Cloud Platform project that the job belongs to.", 360 "description": "The ID of the Cloud Platform project that the job belongs to.",
615 "location": "path", 361 "location": "path",
616 "required": true, 362 "required": true,
617 "type": "string" 363 "type": "string"
618 }, 364 },
619 "location": {
620 "description": "The location that co ntains this job.",
621 "location": "path",
622 "required": true,
623 "type": "string"
624 },
625 "view": { 365 "view": {
626 "description": "The level of informa tion requested in response.", 366 "description": "The level of informa tion requested in response.",
627 "enum": [ 367 "enum": [
628 "JOB_VIEW_UNKNOWN", 368 "JOB_VIEW_UNKNOWN",
629 "JOB_VIEW_SUMMARY", 369 "JOB_VIEW_SUMMARY",
630 "JOB_VIEW_ALL" 370 "JOB_VIEW_ALL"
631 ], 371 ],
632 "location": "query", 372 "location": "query",
633 "type": "string" 373 "type": "string"
634 }, 374 },
375 "location": {
376 "description": "The location that co ntains this job.",
377 "location": "path",
378 "required": true,
379 "type": "string"
380 },
635 "replaceJobId": { 381 "replaceJobId": {
636 "description": "Deprecated. This fie ld is now in the Job message.", 382 "description": "Deprecated. This fie ld is now in the Job message.",
637 "location": "query", 383 "location": "query",
638 "type": "string" 384 "type": "string"
639 } 385 }
640 }, 386 },
641 "path": "v1b3/projects/{projectId}/locations /{location}/jobs", 387 "path": "v1b3/projects/{projectId}/locations /{location}/jobs",
642 "request": { 388 "request": {
643 "$ref": "Job" 389 "$ref": "Job"
644 }, 390 },
645 "response": { 391 "response": {
646 "$ref": "Job" 392 "$ref": "Job"
647 }, 393 },
648 "scopes": [ 394 "scopes": [
649 "https://www.googleapis.com/auth/cloud-p latform", 395 "https://www.googleapis.com/auth/cloud-p latform",
650 "https://www.googleapis.com/auth/userinf o.email" 396 "https://www.googleapis.com/auth/userinf o.email"
651 ] 397 ]
652 },
653 "get": {
654 "description": "Gets the state of the specif ied Cloud Dataflow job.",
655 "httpMethod": "GET",
656 "id": "dataflow.projects.locations.jobs.get" ,
657 "parameterOrder": [
658 "projectId",
659 "location",
660 "jobId"
661 ],
662 "parameters": {
663 "projectId": {
664 "description": "The ID of the Cloud Platform project that the job belongs to.",
665 "location": "path",
666 "required": true,
667 "type": "string"
668 },
669 "location": {
670 "description": "The location that co ntains this job.",
671 "location": "path",
672 "required": true,
673 "type": "string"
674 },
675 "jobId": {
676 "description": "The job ID.",
677 "location": "path",
678 "required": true,
679 "type": "string"
680 },
681 "view": {
682 "description": "The level of informa tion requested in response.",
683 "enum": [
684 "JOB_VIEW_UNKNOWN",
685 "JOB_VIEW_SUMMARY",
686 "JOB_VIEW_ALL"
687 ],
688 "location": "query",
689 "type": "string"
690 }
691 },
692 "path": "v1b3/projects/{projectId}/locations /{location}/jobs/{jobId}",
693 "response": {
694 "$ref": "Job"
695 },
696 "scopes": [
697 "https://www.googleapis.com/auth/cloud-p latform",
698 "https://www.googleapis.com/auth/userinf o.email"
699 ]
700 },
701 "update": {
702 "description": "Updates the state of an exis ting Cloud Dataflow job.",
703 "httpMethod": "PUT",
704 "id": "dataflow.projects.locations.jobs.upda te",
705 "parameterOrder": [
706 "projectId",
707 "location",
708 "jobId"
709 ],
710 "parameters": {
711 "projectId": {
712 "description": "The ID of the Cloud Platform project that the job belongs to.",
713 "location": "path",
714 "required": true,
715 "type": "string"
716 },
717 "location": {
718 "description": "The location that co ntains this job.",
719 "location": "path",
720 "required": true,
721 "type": "string"
722 },
723 "jobId": {
724 "description": "The job ID.",
725 "location": "path",
726 "required": true,
727 "type": "string"
728 }
729 },
730 "path": "v1b3/projects/{projectId}/locations /{location}/jobs/{jobId}",
731 "request": {
732 "$ref": "Job"
733 },
734 "response": {
735 "$ref": "Job"
736 },
737 "scopes": [
738 "https://www.googleapis.com/auth/cloud-p latform",
739 "https://www.googleapis.com/auth/userinf o.email"
740 ]
741 },
742 "list": {
743 "description": "List the jobs of a project." ,
744 "httpMethod": "GET",
745 "id": "dataflow.projects.locations.jobs.list ",
746 "parameterOrder": [
747 "projectId",
748 "location"
749 ],
750 "parameters": {
751 "projectId": {
752 "description": "The project which ow ns the jobs.",
753 "location": "path",
754 "required": true,
755 "type": "string"
756 },
757 "location": {
758 "description": "The location that co ntains this job.",
759 "location": "path",
760 "required": true,
761 "type": "string"
762 },
763 "filter": {
764 "description": "The kind of filter t o use.",
765 "enum": [
766 "UNKNOWN",
767 "ALL",
768 "TERMINATED",
769 "ACTIVE"
770 ],
771 "location": "query",
772 "type": "string"
773 },
774 "view": {
775 "description": "Level of information requested in response. Default is `JOB_VIEW_SUMMARY`.",
776 "enum": [
777 "JOB_VIEW_UNKNOWN",
778 "JOB_VIEW_SUMMARY",
779 "JOB_VIEW_ALL"
780 ],
781 "location": "query",
782 "type": "string"
783 },
784 "pageSize": {
785 "description": "If there are many jo bs, limit response to at most this many. The actual number of jobs returned will be the lesser of max_responses and an unspecified server-defined limit.",
786 "format": "int32",
787 "location": "query",
788 "type": "integer"
789 },
790 "pageToken": {
791 "description": "Set this to the 'nex t_page_token' field of a previous response to request additional results in a lo ng list.",
792 "location": "query",
793 "type": "string"
794 }
795 },
796 "path": "v1b3/projects/{projectId}/locations /{location}/jobs",
797 "response": {
798 "$ref": "ListJobsResponse"
799 },
800 "scopes": [
801 "https://www.googleapis.com/auth/cloud-p latform",
802 "https://www.googleapis.com/auth/userinf o.email"
803 ]
804 },
805 "getMetrics": {
806 "description": "Request the job status.",
807 "httpMethod": "GET",
808 "id": "dataflow.projects.locations.jobs.getM etrics",
809 "parameterOrder": [
810 "projectId",
811 "location",
812 "jobId"
813 ],
814 "parameters": {
815 "projectId": {
816 "description": "A project id.",
817 "location": "path",
818 "required": true,
819 "type": "string"
820 },
821 "location": {
822 "description": "The location which c ontains the job specified by job_id.",
823 "location": "path",
824 "required": true,
825 "type": "string"
826 },
827 "jobId": {
828 "description": "The job to get messa ges for.",
829 "location": "path",
830 "required": true,
831 "type": "string"
832 },
833 "startTime": {
834 "description": "Return only metric d ata that has changed since this time. Default is to return all information about all metrics for the job.",
835 "location": "query",
836 "type": "string"
837 }
838 },
839 "path": "v1b3/projects/{projectId}/locations /{location}/jobs/{jobId}/metrics",
840 "response": {
841 "$ref": "JobMetrics"
842 },
843 "scopes": [
844 "https://www.googleapis.com/auth/cloud-p latform",
845 "https://www.googleapis.com/auth/userinf o.email"
846 ]
847 } 398 }
848 }, 399 },
849 "resources": { 400 "resources": {
850 "messages": { 401 "workItems": {
851 "methods": { 402 "methods": {
852 "list": { 403 "lease": {
853 "description": "Request the job stat us.", 404 "description": "Leases a dataflow Wo rkItem to run.",
854 "httpMethod": "GET", 405 "httpMethod": "POST",
855 "id": "dataflow.projects.locations.j obs.messages.list", 406 "id": "dataflow.projects.locations.j obs.workItems.lease",
856 "parameterOrder": [ 407 "parameterOrder": [
857 "projectId", 408 "projectId",
858 "location", 409 "location",
859 "jobId" 410 "jobId"
860 ], 411 ],
861 "parameters": { 412 "parameters": {
862 "projectId": { 413 "location": {
863 "description": "A project id .", 414 "description": "The location which contains the WorkItem's job.",
864 "location": "path", 415 "location": "path",
865 "required": true, 416 "required": true,
866 "type": "string" 417 "type": "string"
867 }, 418 },
868 "location": { 419 "projectId": {
869 "description": "The location which contains the job specified by job_id.", 420 "description": "Identifies t he project this worker belongs to.",
870 "location": "path", 421 "location": "path",
871 "required": true, 422 "required": true,
872 "type": "string" 423 "type": "string"
873 }, 424 },
874 "jobId": { 425 "jobId": {
875 "description": "The job to g et messages about.", 426 "description": "Identifies t he workflow job this worker belongs to.",
876 "location": "path", 427 "location": "path",
877 "required": true, 428 "required": true,
878 "type": "string" 429 "type": "string"
879 },
880 "minimumImportance": {
881 "description": "Filter to on ly get messages with importance >= level",
882 "enum": [
883 "JOB_MESSAGE_IMPORTANCE_ UNKNOWN",
884 "JOB_MESSAGE_DEBUG",
885 "JOB_MESSAGE_DETAILED",
886 "JOB_MESSAGE_BASIC",
887 "JOB_MESSAGE_WARNING",
888 "JOB_MESSAGE_ERROR"
889 ],
890 "location": "query",
891 "type": "string"
892 },
893 "pageSize": {
894 "description": "If specified , determines the maximum number of messages to return. If unspecified, the servi ce may choose an appropriate default, or may return an arbitrarily large number of results.",
895 "format": "int32",
896 "location": "query",
897 "type": "integer"
898 },
899 "pageToken": {
900 "description": "If supplied, this should be the value of next_page_token returned by an earlier call. This w ill cause the next page of results to be returned.",
901 "location": "query",
902 "type": "string"
903 },
904 "startTime": {
905 "description": "If specified , return only messages with timestamps >= start_time. The default is the job cre ation time (i.e. beginning of messages).",
906 "location": "query",
907 "type": "string"
908 },
909 "endTime": {
910 "description": "Return only messages with timestamps < end_time. The default is now (i.e. return up to the l atest messages available).",
911 "location": "query",
912 "type": "string"
913 } 430 }
914 }, 431 },
915 "path": "v1b3/projects/{projectId}/l ocations/{location}/jobs/{jobId}/messages", 432 "path": "v1b3/projects/{projectId}/l ocations/{location}/jobs/{jobId}/workItems:lease",
433 "request": {
434 "$ref": "LeaseWorkItemRequest"
435 },
916 "response": { 436 "response": {
917 "$ref": "ListJobMessagesResponse " 437 "$ref": "LeaseWorkItemResponse"
918 }, 438 },
919 "scopes": [ 439 "scopes": [
920 "https://www.googleapis.com/auth /cloud-platform", 440 "https://www.googleapis.com/auth /cloud-platform",
921 "https://www.googleapis.com/auth /userinfo.email" 441 "https://www.googleapis.com/auth /userinfo.email"
922 ] 442 ]
923 } 443 },
924 }
925 },
926 "workItems": {
927 "methods": {
928 "reportStatus": { 444 "reportStatus": {
929 "description": "Reports the status o f dataflow WorkItems leased by a worker.", 445 "description": "Reports the status o f dataflow WorkItems leased by a worker.",
930 "httpMethod": "POST", 446 "httpMethod": "POST",
931 "id": "dataflow.projects.locations.j obs.workItems.reportStatus", 447 "id": "dataflow.projects.locations.j obs.workItems.reportStatus",
932 "parameterOrder": [ 448 "parameterOrder": [
933 "projectId", 449 "projectId",
934 "location", 450 "location",
935 "jobId" 451 "jobId"
936 ], 452 ],
937 "parameters": { 453 "parameters": {
938 "projectId": { 454 "projectId": {
939 "description": "The project which owns the WorkItem's job.", 455 "description": "The project which owns the WorkItem's job.",
940 "location": "path", 456 "location": "path",
941 "required": true, 457 "required": true,
942 "type": "string" 458 "type": "string"
943 }, 459 },
460 "jobId": {
461 "description": "The job whic h the WorkItem is part of.",
462 "location": "path",
463 "required": true,
464 "type": "string"
465 },
944 "location": { 466 "location": {
945 "description": "The location which contains the WorkItem's job.", 467 "description": "The location which contains the WorkItem's job.",
946 "location": "path", 468 "location": "path",
947 "required": true, 469 "required": true,
948 "type": "string" 470 "type": "string"
949 },
950 "jobId": {
951 "description": "The job whic h the WorkItem is part of.",
952 "location": "path",
953 "required": true,
954 "type": "string"
955 } 471 }
956 }, 472 },
957 "path": "v1b3/projects/{projectId}/l ocations/{location}/jobs/{jobId}/workItems:reportStatus", 473 "path": "v1b3/projects/{projectId}/l ocations/{location}/jobs/{jobId}/workItems:reportStatus",
958 "request": { 474 "request": {
959 "$ref": "ReportWorkItemStatusReq uest" 475 "$ref": "ReportWorkItemStatusReq uest"
960 }, 476 },
961 "response": { 477 "response": {
962 "$ref": "ReportWorkItemStatusRes ponse" 478 "$ref": "ReportWorkItemStatusRes ponse"
963 }, 479 },
964 "scopes": [ 480 "scopes": [
965 "https://www.googleapis.com/auth /cloud-platform", 481 "https://www.googleapis.com/auth /cloud-platform",
966 "https://www.googleapis.com/auth /userinfo.email" 482 "https://www.googleapis.com/auth /userinfo.email"
967 ] 483 ]
968 }, 484 }
969 "lease": { 485 }
970 "description": "Leases a dataflow Wo rkItem to run.", 486 },
971 "httpMethod": "POST", 487 "messages": {
972 "id": "dataflow.projects.locations.j obs.workItems.lease", 488 "methods": {
489 "list": {
490 "description": "Request the job stat us.",
491 "httpMethod": "GET",
492 "id": "dataflow.projects.locations.j obs.messages.list",
973 "parameterOrder": [ 493 "parameterOrder": [
974 "projectId", 494 "projectId",
975 "location", 495 "location",
976 "jobId" 496 "jobId"
977 ], 497 ],
978 "parameters": { 498 "parameters": {
979 "projectId": { 499 "jobId": {
980 "description": "Identifies t he project this worker belongs to.", 500 "description": "The job to g et messages about.",
981 "location": "path", 501 "location": "path",
982 "required": true, 502 "required": true,
983 "type": "string" 503 "type": "string"
984 }, 504 },
985 "location": { 505 "projectId": {
986 "description": "The location which contains the WorkItem's job.", 506 "description": "A project id .",
987 "location": "path", 507 "location": "path",
988 "required": true, 508 "required": true,
989 "type": "string" 509 "type": "string"
990 }, 510 },
991 "jobId": { 511 "endTime": {
992 "description": "Identifies t he workflow job this worker belongs to.", 512 "description": "Return only messages with timestamps < end_time. The default is now\n(i.e. return up to the latest messages available).",
513 "format": "google-datetime",
514 "location": "query",
515 "type": "string"
516 },
517 "location": {
518 "description": "The location which contains the job specified by job_id.",
993 "location": "path", 519 "location": "path",
994 "required": true, 520 "required": true,
995 "type": "string" 521 "type": "string"
522 },
523 "startTime": {
524 "description": "If specified , return only messages with timestamps >= start_time.\nThe default is the job cr eation time (i.e. beginning of messages).",
525 "format": "google-datetime",
526 "location": "query",
527 "type": "string"
528 },
529 "pageToken": {
530 "description": "If supplied, this should be the value of next_page_token returned\nby an earlier call. This will cause the next page of results to\nbe returned.",
531 "location": "query",
532 "type": "string"
533 },
534 "pageSize": {
535 "description": "If specified , determines the maximum number of messages to\nreturn. If unspecified, the ser vice may choose an appropriate\ndefault, or may return an arbitrarily large numb er of results.",
536 "format": "int32",
537 "location": "query",
538 "type": "integer"
539 },
540 "minimumImportance": {
541 "description": "Filter to on ly get messages with importance >= level",
542 "enum": [
543 "JOB_MESSAGE_IMPORTANCE_ UNKNOWN",
544 "JOB_MESSAGE_DEBUG",
545 "JOB_MESSAGE_DETAILED",
546 "JOB_MESSAGE_BASIC",
547 "JOB_MESSAGE_WARNING",
548 "JOB_MESSAGE_ERROR"
549 ],
550 "location": "query",
551 "type": "string"
996 } 552 }
997 }, 553 },
998 "path": "v1b3/projects/{projectId}/l ocations/{location}/jobs/{jobId}/workItems:lease", 554 "path": "v1b3/projects/{projectId}/l ocations/{location}/jobs/{jobId}/messages",
999 "request": {
1000 "$ref": "LeaseWorkItemRequest"
1001 },
1002 "response": { 555 "response": {
1003 "$ref": "LeaseWorkItemResponse" 556 "$ref": "ListJobMessagesResponse "
1004 }, 557 },
1005 "scopes": [ 558 "scopes": [
1006 "https://www.googleapis.com/auth /cloud-platform", 559 "https://www.googleapis.com/auth /cloud-platform",
1007 "https://www.googleapis.com/auth /userinfo.email" 560 "https://www.googleapis.com/auth /userinfo.email"
1008 ] 561 ]
1009 } 562 }
1010 } 563 }
1011 } 564 }
1012 } 565 }
1013 } 566 }
(...skipping 22 matching lines...) Expand all
1036 }, 589 },
1037 "response": { 590 "response": {
1038 "$ref": "Job" 591 "$ref": "Job"
1039 }, 592 },
1040 "scopes": [ 593 "scopes": [
1041 "https://www.googleapis.com/auth/cloud-platform" , 594 "https://www.googleapis.com/auth/cloud-platform" ,
1042 "https://www.googleapis.com/auth/userinfo.email" 595 "https://www.googleapis.com/auth/userinfo.email"
1043 ] 596 ]
1044 } 597 }
1045 } 598 }
599 },
600 "jobs": {
601 "methods": {
602 "getMetrics": {
603 "description": "Request the job status.",
604 "httpMethod": "GET",
605 "id": "dataflow.projects.jobs.getMetrics",
606 "parameterOrder": [
607 "projectId",
608 "jobId"
609 ],
610 "parameters": {
611 "projectId": {
612 "description": "A project id.",
613 "location": "path",
614 "required": true,
615 "type": "string"
616 },
617 "jobId": {
618 "description": "The job to get messages for. ",
619 "location": "path",
620 "required": true,
621 "type": "string"
622 },
623 "location": {
624 "description": "The location which contains the job specified by job_id.",
625 "location": "query",
626 "type": "string"
627 },
628 "startTime": {
629 "description": "Return only metric data that has changed since this time.\nDefault is to return all information about all me trics for the job.",
630 "format": "google-datetime",
631 "location": "query",
632 "type": "string"
633 }
634 },
635 "path": "v1b3/projects/{projectId}/jobs/{jobId}/metr ics",
636 "response": {
637 "$ref": "JobMetrics"
638 },
639 "scopes": [
640 "https://www.googleapis.com/auth/cloud-platform" ,
641 "https://www.googleapis.com/auth/userinfo.email"
642 ]
643 },
644 "get": {
645 "description": "Gets the state of the specified Clou d Dataflow job.",
646 "httpMethod": "GET",
647 "id": "dataflow.projects.jobs.get",
648 "parameterOrder": [
649 "projectId",
650 "jobId"
651 ],
652 "parameters": {
653 "projectId": {
654 "description": "The ID of the Cloud Platform project that the job belongs to.",
655 "location": "path",
656 "required": true,
657 "type": "string"
658 },
659 "jobId": {
660 "description": "The job ID.",
661 "location": "path",
662 "required": true,
663 "type": "string"
664 },
665 "view": {
666 "description": "The level of information req uested in response.",
667 "enum": [
668 "JOB_VIEW_UNKNOWN",
669 "JOB_VIEW_SUMMARY",
670 "JOB_VIEW_ALL"
671 ],
672 "location": "query",
673 "type": "string"
674 },
675 "location": {
676 "description": "The location that contains t his job.",
677 "location": "query",
678 "type": "string"
679 }
680 },
681 "path": "v1b3/projects/{projectId}/jobs/{jobId}",
682 "response": {
683 "$ref": "Job"
684 },
685 "scopes": [
686 "https://www.googleapis.com/auth/cloud-platform" ,
687 "https://www.googleapis.com/auth/userinfo.email"
688 ]
689 },
690 "list": {
691 "description": "List the jobs of a project.",
692 "httpMethod": "GET",
693 "id": "dataflow.projects.jobs.list",
694 "parameterOrder": [
695 "projectId"
696 ],
697 "parameters": {
698 "projectId": {
699 "description": "The project which owns the j obs.",
700 "location": "path",
701 "required": true,
702 "type": "string"
703 },
704 "filter": {
705 "description": "The kind of filter to use.",
706 "enum": [
707 "UNKNOWN",
708 "ALL",
709 "TERMINATED",
710 "ACTIVE"
711 ],
712 "location": "query",
713 "type": "string"
714 },
715 "location": {
716 "description": "The location that contains t his job.",
717 "location": "query",
718 "type": "string"
719 },
720 "pageToken": {
721 "description": "Set this to the 'next_page_t oken' field of a previous response\nto request additional results in a long list .",
722 "location": "query",
723 "type": "string"
724 },
725 "pageSize": {
726 "description": "If there are many jobs, limi t response to at most this many.\nThe actual number of jobs returned will be the lesser of max_responses\nand an unspecified server-defined limit.",
727 "format": "int32",
728 "location": "query",
729 "type": "integer"
730 },
731 "view": {
732 "description": "Level of information request ed in response. Default is `JOB_VIEW_SUMMARY`.",
733 "enum": [
734 "JOB_VIEW_UNKNOWN",
735 "JOB_VIEW_SUMMARY",
736 "JOB_VIEW_ALL"
737 ],
738 "location": "query",
739 "type": "string"
740 }
741 },
742 "path": "v1b3/projects/{projectId}/jobs",
743 "response": {
744 "$ref": "ListJobsResponse"
745 },
746 "scopes": [
747 "https://www.googleapis.com/auth/cloud-platform" ,
748 "https://www.googleapis.com/auth/userinfo.email"
749 ]
750 },
751 "update": {
752 "description": "Updates the state of an existing Clo ud Dataflow job.",
753 "httpMethod": "PUT",
754 "id": "dataflow.projects.jobs.update",
755 "parameterOrder": [
756 "projectId",
757 "jobId"
758 ],
759 "parameters": {
760 "location": {
761 "description": "The location that contains t his job.",
762 "location": "query",
763 "type": "string"
764 },
765 "projectId": {
766 "description": "The ID of the Cloud Platform project that the job belongs to.",
767 "location": "path",
768 "required": true,
769 "type": "string"
770 },
771 "jobId": {
772 "description": "The job ID.",
773 "location": "path",
774 "required": true,
775 "type": "string"
776 }
777 },
778 "path": "v1b3/projects/{projectId}/jobs/{jobId}",
779 "request": {
780 "$ref": "Job"
781 },
782 "response": {
783 "$ref": "Job"
784 },
785 "scopes": [
786 "https://www.googleapis.com/auth/cloud-platform" ,
787 "https://www.googleapis.com/auth/userinfo.email"
788 ]
789 },
790 "create": {
791 "description": "Creates a Cloud Dataflow job.",
792 "httpMethod": "POST",
793 "id": "dataflow.projects.jobs.create",
794 "parameterOrder": [
795 "projectId"
796 ],
797 "parameters": {
798 "location": {
799 "description": "The location that contains t his job.",
800 "location": "query",
801 "type": "string"
802 },
803 "replaceJobId": {
804 "description": "Deprecated. This field is no w in the Job message.",
805 "location": "query",
806 "type": "string"
807 },
808 "projectId": {
809 "description": "The ID of the Cloud Platform project that the job belongs to.",
810 "location": "path",
811 "required": true,
812 "type": "string"
813 },
814 "view": {
815 "description": "The level of information req uested in response.",
816 "enum": [
817 "JOB_VIEW_UNKNOWN",
818 "JOB_VIEW_SUMMARY",
819 "JOB_VIEW_ALL"
820 ],
821 "location": "query",
822 "type": "string"
823 }
824 },
825 "path": "v1b3/projects/{projectId}/jobs",
826 "request": {
827 "$ref": "Job"
828 },
829 "response": {
830 "$ref": "Job"
831 },
832 "scopes": [
833 "https://www.googleapis.com/auth/cloud-platform" ,
834 "https://www.googleapis.com/auth/userinfo.email"
835 ]
836 }
837 },
838 "resources": {
839 "debug": {
840 "methods": {
841 "getConfig": {
842 "description": "Get encoded debug configurat ion for component. Not cacheable.",
843 "httpMethod": "POST",
844 "id": "dataflow.projects.jobs.debug.getConfi g",
845 "parameterOrder": [
846 "projectId",
847 "jobId"
848 ],
849 "parameters": {
850 "projectId": {
851 "description": "The project id.",
852 "location": "path",
853 "required": true,
854 "type": "string"
855 },
856 "jobId": {
857 "description": "The job id.",
858 "location": "path",
859 "required": true,
860 "type": "string"
861 }
862 },
863 "path": "v1b3/projects/{projectId}/jobs/{job Id}/debug/getConfig",
864 "request": {
865 "$ref": "GetDebugConfigRequest"
866 },
867 "response": {
868 "$ref": "GetDebugConfigResponse"
869 },
870 "scopes": [
871 "https://www.googleapis.com/auth/cloud-p latform",
872 "https://www.googleapis.com/auth/userinf o.email"
873 ]
874 },
875 "sendCapture": {
876 "description": "Send encoded debug capture d ata for component.",
877 "httpMethod": "POST",
878 "id": "dataflow.projects.jobs.debug.sendCapt ure",
879 "parameterOrder": [
880 "projectId",
881 "jobId"
882 ],
883 "parameters": {
884 "projectId": {
885 "description": "The project id.",
886 "location": "path",
887 "required": true,
888 "type": "string"
889 },
890 "jobId": {
891 "description": "The job id.",
892 "location": "path",
893 "required": true,
894 "type": "string"
895 }
896 },
897 "path": "v1b3/projects/{projectId}/jobs/{job Id}/debug/sendCapture",
898 "request": {
899 "$ref": "SendDebugCaptureRequest"
900 },
901 "response": {
902 "$ref": "SendDebugCaptureResponse"
903 },
904 "scopes": [
905 "https://www.googleapis.com/auth/cloud-p latform",
906 "https://www.googleapis.com/auth/userinf o.email"
907 ]
908 }
909 }
910 },
911 "workItems": {
912 "methods": {
913 "reportStatus": {
914 "description": "Reports the status of datafl ow WorkItems leased by a worker.",
915 "httpMethod": "POST",
916 "id": "dataflow.projects.jobs.workItems.repo rtStatus",
917 "parameterOrder": [
918 "projectId",
919 "jobId"
920 ],
921 "parameters": {
922 "projectId": {
923 "description": "The project which ow ns the WorkItem's job.",
924 "location": "path",
925 "required": true,
926 "type": "string"
927 },
928 "jobId": {
929 "description": "The job which the Wo rkItem is part of.",
930 "location": "path",
931 "required": true,
932 "type": "string"
933 }
934 },
935 "path": "v1b3/projects/{projectId}/jobs/{job Id}/workItems:reportStatus",
936 "request": {
937 "$ref": "ReportWorkItemStatusRequest"
938 },
939 "response": {
940 "$ref": "ReportWorkItemStatusResponse"
941 },
942 "scopes": [
943 "https://www.googleapis.com/auth/cloud-p latform",
944 "https://www.googleapis.com/auth/userinf o.email"
945 ]
946 },
947 "lease": {
948 "description": "Leases a dataflow WorkItem t o run.",
949 "httpMethod": "POST",
950 "id": "dataflow.projects.jobs.workItems.leas e",
951 "parameterOrder": [
952 "projectId",
953 "jobId"
954 ],
955 "parameters": {
956 "projectId": {
957 "description": "Identifies the proje ct this worker belongs to.",
958 "location": "path",
959 "required": true,
960 "type": "string"
961 },
962 "jobId": {
963 "description": "Identifies the workf low job this worker belongs to.",
964 "location": "path",
965 "required": true,
966 "type": "string"
967 }
968 },
969 "path": "v1b3/projects/{projectId}/jobs/{job Id}/workItems:lease",
970 "request": {
971 "$ref": "LeaseWorkItemRequest"
972 },
973 "response": {
974 "$ref": "LeaseWorkItemResponse"
975 },
976 "scopes": [
977 "https://www.googleapis.com/auth/cloud-p latform",
978 "https://www.googleapis.com/auth/userinf o.email"
979 ]
980 }
981 }
982 },
983 "messages": {
984 "methods": {
985 "list": {
986 "description": "Request the job status.",
987 "httpMethod": "GET",
988 "id": "dataflow.projects.jobs.messages.list" ,
989 "parameterOrder": [
990 "projectId",
991 "jobId"
992 ],
993 "parameters": {
994 "minimumImportance": {
995 "description": "Filter to only get m essages with importance >= level",
996 "enum": [
997 "JOB_MESSAGE_IMPORTANCE_UNKNOWN" ,
998 "JOB_MESSAGE_DEBUG",
999 "JOB_MESSAGE_DETAILED",
1000 "JOB_MESSAGE_BASIC",
1001 "JOB_MESSAGE_WARNING",
1002 "JOB_MESSAGE_ERROR"
1003 ],
1004 "location": "query",
1005 "type": "string"
1006 },
1007 "jobId": {
1008 "description": "The job to get messa ges about.",
1009 "location": "path",
1010 "required": true,
1011 "type": "string"
1012 },
1013 "projectId": {
1014 "description": "A project id.",
1015 "location": "path",
1016 "required": true,
1017 "type": "string"
1018 },
1019 "location": {
1020 "description": "The location which c ontains the job specified by job_id.",
1021 "location": "query",
1022 "type": "string"
1023 },
1024 "endTime": {
1025 "description": "Return only messages with timestamps < end_time. The default is now\n(i.e. return up to the latest m essages available).",
1026 "format": "google-datetime",
1027 "location": "query",
1028 "type": "string"
1029 },
1030 "startTime": {
1031 "description": "If specified, return only messages with timestamps >= start_time.\nThe default is the job creation t ime (i.e. beginning of messages).",
1032 "format": "google-datetime",
1033 "location": "query",
1034 "type": "string"
1035 },
1036 "pageToken": {
1037 "description": "If supplied, this sh ould be the value of next_page_token returned\nby an earlier call. This will cau se the next page of results to\nbe returned.",
1038 "location": "query",
1039 "type": "string"
1040 },
1041 "pageSize": {
1042 "description": "If specified, determ ines the maximum number of messages to\nreturn. If unspecified, the service may choose an appropriate\ndefault, or may return an arbitrarily large number of re sults.",
1043 "format": "int32",
1044 "location": "query",
1045 "type": "integer"
1046 }
1047 },
1048 "path": "v1b3/projects/{projectId}/jobs/{job Id}/messages",
1049 "response": {
1050 "$ref": "ListJobMessagesResponse"
1051 },
1052 "scopes": [
1053 "https://www.googleapis.com/auth/cloud-p latform",
1054 "https://www.googleapis.com/auth/userinf o.email"
1055 ]
1056 }
1057 }
1058 }
1059 }
1046 } 1060 }
1047 } 1061 }
1048 } 1062 }
1049 }, 1063 },
1050 "revision": "20161004", 1064 "revision": "20170210",
1051 "rootUrl": "https://dataflow.googleapis.com/", 1065 "rootUrl": "https://dataflow.googleapis.com/",
1052 "schemas": { 1066 "schemas": {
1053 "GetDebugConfigRequest": { 1067 "WorkItemStatus": {
1054 "description": "Request to get updated debug configuration for compo nent.", 1068 "description": "Conveys a worker's progress through the work describ ed by a WorkItem.",
1055 "id": "GetDebugConfigRequest", 1069 "id": "WorkItemStatus",
1056 "properties": { 1070 "properties": {
1057 "workerId": { 1071 "counterUpdates": {
1058 "description": "The worker id, i.e., VM hostname.", 1072 "description": "Worker output counters for this WorkItem.",
1059 "type": "string"
1060 },
1061 "componentId": {
1062 "description": "The internal component id for which debug co nfiguration is requested.",
1063 "type": "string"
1064 }
1065 },
1066 "type": "object"
1067 },
1068 "GetDebugConfigResponse": {
1069 "description": "Response to a get debug configuration request.",
1070 "id": "GetDebugConfigResponse",
1071 "properties": {
1072 "config": {
1073 "description": "The encoded debug configuration for the requ ested component.",
1074 "type": "string"
1075 }
1076 },
1077 "type": "object"
1078 },
1079 "SendDebugCaptureRequest": {
1080 "description": "Request to send encoded debug information.",
1081 "id": "SendDebugCaptureRequest",
1082 "properties": {
1083 "workerId": {
1084 "description": "The worker id, i.e., VM hostname.",
1085 "type": "string"
1086 },
1087 "componentId": {
1088 "description": "The internal component id for which debug in formation is sent.",
1089 "type": "string"
1090 },
1091 "data": {
1092 "description": "The encoded debug information.",
1093 "type": "string"
1094 }
1095 },
1096 "type": "object"
1097 },
1098 "SendDebugCaptureResponse": {
1099 "description": "Response to a send capture request. nothing",
1100 "id": "SendDebugCaptureResponse",
1101 "type": "object"
1102 },
1103 "Job": {
1104 "description": "Defines a job to be run by the Cloud Dataflow servic e.",
1105 "id": "Job",
1106 "properties": {
1107 "id": {
1108 "description": "The unique ID of this job. This field is set by the Cloud Dataflow service when the Job is created, and is immutable for the life of the job.",
1109 "type": "string"
1110 },
1111 "projectId": {
1112 "description": "The ID of the Cloud Platform project that th e job belongs to.",
1113 "type": "string"
1114 },
1115 "name": {
1116 "description": "The user-specified Cloud Dataflow job name. Only one Job with a given name may exist in a project at any given time. If a ca ller attempts to create a Job with the same name as an already-existing Job, the attempt returns the existing Job. The name must match the regular expression `[ a-z]([-a-z0-9]{0,38}[a-z0-9])?`",
1117 "type": "string"
1118 },
1119 "type": {
1120 "description": "The type of Cloud Dataflow job.",
1121 "enum": [
1122 "JOB_TYPE_UNKNOWN",
1123 "JOB_TYPE_BATCH",
1124 "JOB_TYPE_STREAMING"
1125 ],
1126 "type": "string"
1127 },
1128 "environment": {
1129 "$ref": "Environment",
1130 "description": "The environment for the job."
1131 },
1132 "steps": {
1133 "description": "The top-level steps that constitute the enti re job.",
1134 "items": { 1073 "items": {
1135 "$ref": "Step" 1074 "$ref": "CounterUpdate"
1136 }, 1075 },
1137 "type": "array" 1076 "type": "array"
1138 }, 1077 },
1139 "currentState": { 1078 "workItemId": {
1140 "description": "The current state of the job. Jobs are creat ed in the `JOB_STATE_STOPPED` state unless otherwise specified. A job in the `JO B_STATE_RUNNING` state may asynchronously enter a terminal state. After a job ha s reached a terminal state, no further state updates may be made. This field may be mutated by the Cloud Dataflow service; callers cannot mutate it.", 1079 "description": "Identifies the WorkItem.",
1141 "enum": [ 1080 "type": "string"
1142 "JOB_STATE_UNKNOWN", 1081 },
1143 "JOB_STATE_STOPPED", 1082 "metricUpdates": {
1144 "JOB_STATE_RUNNING", 1083 "description": "DEPRECATED in favor of counter_updates.",
1145 "JOB_STATE_DONE",
1146 "JOB_STATE_FAILED",
1147 "JOB_STATE_CANCELLED",
1148 "JOB_STATE_UPDATED",
1149 "JOB_STATE_DRAINING",
1150 "JOB_STATE_DRAINED"
1151 ],
1152 "type": "string"
1153 },
1154 "currentStateTime": {
1155 "description": "The timestamp associated with the current st ate.",
1156 "type": "string"
1157 },
1158 "requestedState": {
1159 "description": "The job's requested state. `UpdateJob` may b e used to switch between the `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING` states, by setting requested_state. `UpdateJob` may also be used to directly set a job' s requested state to `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably term inating the job if it has not already reached a terminal state.",
1160 "enum": [
1161 "JOB_STATE_UNKNOWN",
1162 "JOB_STATE_STOPPED",
1163 "JOB_STATE_RUNNING",
1164 "JOB_STATE_DONE",
1165 "JOB_STATE_FAILED",
1166 "JOB_STATE_CANCELLED",
1167 "JOB_STATE_UPDATED",
1168 "JOB_STATE_DRAINING",
1169 "JOB_STATE_DRAINED"
1170 ],
1171 "type": "string"
1172 },
1173 "executionInfo": {
1174 "$ref": "JobExecutionInfo",
1175 "description": "Information about how the Cloud Dataflow ser vice will run the job."
1176 },
1177 "createTime": {
1178 "description": "The timestamp when the job was initially cre ated. Immutable and set by the Cloud Dataflow service.",
1179 "type": "string"
1180 },
1181 "replaceJobId": {
1182 "description": "If this job is an update of an existing job, this field is the job ID of the job it replaced. When sending a `CreateJobReque st`, you can update a job by specifying it here. The job named here is stopped, and its intermediate state is transferred to this job.",
1183 "type": "string"
1184 },
1185 "transformNameMapping": {
1186 "additionalProperties": {
1187 "type": "string"
1188 },
1189 "description": "The map of transform name prefixes of the jo b to be replaced to the corresponding name prefixes of the new job.",
1190 "type": "object"
1191 },
1192 "clientRequestId": {
1193 "description": "The client's unique identifier of the job, r e-used across retried attempts. If this field is set, the service will ensure it s uniqueness. The request to create a job will fail if the service has knowledge of a previously submitted job with the same client's ID and job name. The calle r may use this field to ensure idempotence of job creation across retried attemp ts to create a job. By default, the field is empty and, in that case, the servic e ignores it.",
1194 "type": "string"
1195 },
1196 "replacedByJobId": {
1197 "description": "If another job is an update of this job (and thus, this job is in `JOB_STATE_UPDATED`), this field contains the ID of that j ob.",
1198 "type": "string"
1199 },
1200 "tempFiles": {
1201 "description": "A set of files the system should be aware of that are used for temporary storage. These temporary files will be removed on j ob completion. No duplicates are allowed. No file patterns are supported. The su pported files are: Google Cloud Storage: storage.googleapis.com/{bucket}/{object } bucket.storage.googleapis.com/{object}",
1202 "items": { 1084 "items": {
1203 "type": "string" 1085 "$ref": "MetricUpdate"
1204 }, 1086 },
1205 "type": "array" 1087 "type": "array"
1206 }, 1088 },
1207 "labels": { 1089 "errors": {
1208 "additionalProperties": { 1090 "description": "Specifies errors which occurred during proce ssing. If errors are\nprovided, and completed = true, then the WorkItem is cons idered\nto have failed.",
1209 "type": "string"
1210 },
1211 "description": "User-defined labels for this job. The labels map can contain no more than 64 entries. Entries of the labels map are UTF8 str ings that comply with the following restrictions: * Keys must conform to regexp: \\p{Ll}\\p{Lo}{0,62} * Values must conform to regexp: [\\p{Ll}\\p{Lo}\\p{N}_-]{ 0,63} * Both keys and values are additionally constrained to be <= 128 bytes in size.",
1212 "type": "object"
1213 },
1214 "location": {
1215 "description": "The location that contains this job.",
1216 "type": "string"
1217 }
1218 },
1219 "type": "object"
1220 },
1221 "Environment": {
1222 "description": "Describes the environment in which a Dataflow Job ru ns.",
1223 "id": "Environment",
1224 "properties": {
1225 "tempStoragePrefix": {
1226 "description": "The prefix of the resources the system shoul d use for temporary storage. The system will append the suffix \"/temp-{JOBNAME} to this resource prefix, where {JOBNAME} is the value of the job_name field. Th e resulting bucket and object prefix is used as the prefix of the resources used to store temporary data needed during the job execution. NOTE: This will overri de the value in taskrunner_settings. The supported resource type is: Google Clou d Storage: storage.googleapis.com/{bucket}/{object} bucket.storage.googleapis.co m/{object}",
1227 "type": "string"
1228 },
1229 "clusterManagerApiService": {
1230 "description": "The type of cluster manager API to use. If u nknown or unspecified, the service will attempt to choose a reasonable default. This should be in the form of the API service name, e.g. \"compute.googleapis.co m\".",
1231 "type": "string"
1232 },
1233 "experiments": {
1234 "description": "The list of experiments to enable.",
1235 "items": { 1091 "items": {
1236 "type": "string" 1092 "$ref": "Status"
1237 }, 1093 },
1238 "type": "array" 1094 "type": "array"
1239 }, 1095 },
1240 "workerPools": { 1096 "dynamicSourceSplit": {
1241 "description": "The worker pools. At least one \"harness\" w orker pool must be specified in order for the job to have workers.", 1097 "$ref": "DynamicSourceSplit",
1098 "description": "See documentation of stop_position."
1099 },
1100 "sourceOperationResponse": {
1101 "$ref": "SourceOperationResponse",
1102 "description": "If the work item represented a SourceOperati onRequest, and the work\nis completed, contains the result of the operation."
1103 },
1104 "progress": {
1105 "$ref": "ApproximateProgress",
1106 "description": "DEPRECATED in favor of reported_progress."
1107 },
1108 "requestedLeaseDuration": {
1109 "description": "Amount of time the worker requests for its l ease.",
1110 "format": "google-duration",
1111 "type": "string"
1112 },
1113 "reportIndex": {
1114 "description": "The report index. When a WorkItem is leased , the lease will\ncontain an initial report index. When a WorkItem's status is\ nreported to the system, the report should be sent with\nthat report index, and the response will contain the index the\nworker should use for the next report. Reports received with\nunexpected index values will be rejected by the service. \n\nIn order to preserve idempotency, the worker should not alter the\ncontents of a report, even if the worker must submit the same\nreport multiple times befo re getting back a response. The worker\nshould not submit a subsequent report u ntil the response for the\nprevious report had been received from the service.",
1115 "format": "int64",
1116 "type": "string"
1117 },
1118 "stopPosition": {
1119 "$ref": "Position",
1120 "description": "A worker may split an active map task in two parts, \"primary\" and\n\"residual\", continuing to process the primary part an d returning the\nresidual part into the pool of available work.\nThis event is c alled a \"dynamic split\" and is critical to the dynamic\nwork rebalancing featu re. The two obtained sub-tasks are called\n\"parts\" of the split.\nThe parts, i f concatenated, must represent the same input as would\nbe read by the current t ask if the split did not happen.\nThe exact way in which the original task is de composed into the two\nparts is specified either as a position demarcating them\ n(stop_position), or explicitly as two DerivedSources, if this\ntask consumes a user-defined source type (dynamic_source_split).\n\nThe \"current\" task is adju sted as a result of the split: after a task\nwith range [A, B) sends a stop_posi tion update at C, its range is\nconsidered to be [A, C), e.g.:\n* Progress shoul d be interpreted relative to the new range, e.g.\n \"75% completed\" means \"75 % of [A, C) completed\"\n* The worker should interpret proposed_stop_position re lative to the\n new range, e.g. \"split at 68%\" should be interpreted as\n \" split at 68% of [A, C)\".\n* If the worker chooses to split again using stop_pos ition, only\n stop_positions in [A, C) will be accepted.\n* Etc.\ndynamic_sourc e_split has similar semantics: e.g., if a task with\nsource S splits using dynam ic_source_split into {P, R}\n(where P and R must be together equivalent to S), t hen subsequent\nprogress and proposed_stop_position should be interpreted relati ve\nto P, and in a potential subsequent dynamic_source_split into {P', R'},\nP' and R' must be together equivalent to P, etc."
1121 },
1122 "completed": {
1123 "description": "True if the WorkItem was completed (successf ully or unsuccessfully).",
1124 "type": "boolean"
1125 },
1126 "reportedProgress": {
1127 "$ref": "ApproximateReportedProgress",
1128 "description": "The worker's progress through this WorkItem. "
1129 },
1130 "sourceFork": {
1131 "$ref": "SourceFork",
1132 "description": "DEPRECATED in favor of dynamic_source_split. "
1133 }
1134 },
1135 "type": "object"
1136 },
1137 "WorkItemServiceState": {
1138 "description": "The Dataflow service's idea of the current state of a WorkItem\nbeing processed by a worker.",
1139 "id": "WorkItemServiceState",
1140 "properties": {
1141 "leaseExpireTime": {
1142 "description": "Time at which the current lease will expire. ",
1143 "format": "google-datetime",
1144 "type": "string"
1145 },
1146 "metricShortId": {
1147 "description": "The short ids that workers should use in sub sequent metric updates.\nWorkers should strive to use short ids whenever possibl e, but it is ok\nto request the short_id again if a worker lost track of it\n(e. g. if the worker is recovering from a crash).\nNOTE: it is possible that the res ponse may have short ids for a subset\nof the metrics.",
1242 "items": { 1148 "items": {
1243 "$ref": "WorkerPool" 1149 "$ref": "MetricShortId"
1244 }, 1150 },
1245 "type": "array" 1151 "type": "array"
1246 }, 1152 },
1247 "userAgent": { 1153 "nextReportIndex": {
1154 "description": "The index value to use for the next report s ent by the worker.\nNote: If the report call fails for whatever reason, the work er should\nreuse this index for subsequent report attempts.",
1155 "format": "int64",
1156 "type": "string"
1157 },
1158 "suggestedStopPoint": {
1159 "$ref": "ApproximateProgress",
1160 "description": "DEPRECATED in favor of split_request."
1161 },
1162 "splitRequest": {
1163 "$ref": "ApproximateSplitRequest",
1164 "description": "The progress point in the WorkItem where the Dataflow service\nsuggests that the worker truncate the task."
1165 },
1166 "reportStatusInterval": {
1167 "description": "New recommended reporting interval.",
1168 "format": "google-duration",
1169 "type": "string"
1170 },
1171 "suggestedStopPosition": {
1172 "$ref": "Position",
1173 "description": "Obsolete, always empty."
1174 },
1175 "harnessData": {
1248 "additionalProperties": { 1176 "additionalProperties": {
1249 "description": "Properties of the object.", 1177 "description": "Properties of the object.",
1250 "type": "any" 1178 "type": "any"
1251 }, 1179 },
1252 "description": "A description of the process that generated the request.", 1180 "description": "Other data returned by the service, specific to the particular\nworker harness.",
1253 "type": "object" 1181 "type": "object"
1254 }, 1182 }
1255 "version": { 1183 },
1184 "type": "object"
1185 },
1186 "MetricStructuredName": {
1187 "description": "Identifies a metric, by describing the source which generated the\nmetric.",
1188 "id": "MetricStructuredName",
1189 "properties": {
1190 "origin": {
1191 "description": "Origin (namespace) of metric name. May be bl ank for user-define metrics;\nwill be \"dataflow\" for metrics defined by the Da taflow service or SDK.",
1192 "type": "string"
1193 },
1194 "name": {
1195 "description": "Worker-defined metric name.",
1196 "type": "string"
1197 },
1198 "context": {
1199 "additionalProperties": {
1200 "type": "string"
1201 },
1202 "description": "Zero or more labeled fields which identify t he part of the job this\nmetric is associated with, such as the name of a step o r collection.\n\nFor example, built-in counters associated with steps will have\ ncontext['step'] = <step-name>. Counters associated with PCollections\nin the SD K will have context['pcollection'] = <pcollection-name>.",
1203 "type": "object"
1204 }
1205 },
1206 "type": "object"
1207 },
1208 "SeqMapTaskOutputInfo": {
1209 "description": "Information about an output of a SeqMapTask.",
1210 "id": "SeqMapTaskOutputInfo",
1211 "properties": {
1212 "tag": {
1213 "description": "The id of the TupleTag the user code will ta g the output value by.",
1214 "type": "string"
1215 },
1216 "sink": {
1217 "$ref": "Sink",
1218 "description": "The sink to write the output value to."
1219 }
1220 },
1221 "type": "object"
1222 },
1223 "JobExecutionStageInfo": {
1224 "description": "Contains information about how a particular\ngoogle. dataflow.v1beta3.Step will be executed.",
1225 "id": "JobExecutionStageInfo",
1226 "properties": {
1227 "stepName": {
1228 "description": "The steps associated with the execution stag e.\nNote that stages may have several steps, and that a given step\nmight be run by more than one stage.",
1229 "items": {
1230 "type": "string"
1231 },
1232 "type": "array"
1233 }
1234 },
1235 "type": "object"
1236 },
1237 "KeyRangeLocation": {
1238 "description": "Location information for a specific key-range of a s harded computation.\nCurrently we only support UTF-8 character splits to simplif y encoding into\nJSON.",
1239 "id": "KeyRangeLocation",
1240 "properties": {
1241 "deliveryEndpoint": {
1242 "description": "The physical location of this range assignme nt to be used for\nstreaming computation cross-worker message delivery.",
1243 "type": "string"
1244 },
1245 "start": {
1246 "description": "The start (inclusive) of the key range.",
1247 "type": "string"
1248 },
1249 "dataDisk": {
1250 "description": "The name of the data disk where data for thi s range is stored.\nThis name is local to the Google Cloud Platform project and uniquely\nidentifies the disk within that project, for example\n\"myproject-1014 -104817-4c2-harness-0-disk-1\".",
1251 "type": "string"
1252 },
1253 "persistentDirectory": {
1254 "description": "The location of the persistent state for thi s range, as a\npersistent directory in the worker local filesystem.",
1255 "type": "string"
1256 },
1257 "end": {
1258 "description": "The end (exclusive) of the key range.",
1259 "type": "string"
1260 }
1261 },
1262 "type": "object"
1263 },
1264 "SourceGetMetadataRequest": {
1265 "description": "A request to compute the SourceMetadata of a Source. ",
1266 "id": "SourceGetMetadataRequest",
1267 "properties": {
1268 "source": {
1269 "$ref": "Source",
1270 "description": "Specification of the source whose metadata s hould be computed."
1271 }
1272 },
1273 "type": "object"
1274 },
1275 "NameAndKind": {
1276 "description": "Basic metadata about a counter.",
1277 "id": "NameAndKind",
1278 "properties": {
1279 "name": {
1280 "description": "Name of the counter.",
1281 "type": "string"
1282 },
1283 "kind": {
1284 "description": "Counter aggregation kind.",
1285 "enum": [
1286 "INVALID",
1287 "SUM",
1288 "MAX",
1289 "MIN",
1290 "MEAN",
1291 "OR",
1292 "AND",
1293 "SET",
1294 "DISTRIBUTION"
1295 ],
1296 "enumDescriptions": [
1297 "Counter aggregation kind was not set.",
1298 "Aggregated value is the sum of all contributed values." ,
1299 "Aggregated value is the max of all contributed values." ,
1300 "Aggregated value is the min of all contributed values." ,
1301 "Aggregated value is the mean of all contributed values. ",
1302 "Aggregated value represents the logical 'or' of all con tributed values.",
1303 "Aggregated value represents the logical 'and' of all co ntributed values.",
1304 "Aggregated value is a set of unique contributed values. ",
1305 "Aggregated value captures statistics about a distributi on."
1306 ],
1307 "type": "string"
1308 }
1309 },
1310 "type": "object"
1311 },
1312 "SeqMapTask": {
1313 "description": "Describes a particular function to invoke.",
1314 "id": "SeqMapTask",
1315 "properties": {
1316 "userFn": {
1256 "additionalProperties": { 1317 "additionalProperties": {
1257 "description": "Properties of the object.", 1318 "description": "Properties of the object.",
1258 "type": "any" 1319 "type": "any"
1259 }, 1320 },
1260 "description": "A structure describing which components and their versions of the service are required in order to run the job.", 1321 "description": "The user function to invoke.",
1261 "type": "object" 1322 "type": "object"
1262 }, 1323 },
1263 "dataset": { 1324 "name": {
1264 "description": "The dataset for the current project where va rious workflow related tables are stored. The supported resource type is: Google BigQuery: bigquery.googleapis.com/{dataset}", 1325 "description": "The user-provided name of the SeqDo operatio n.",
1265 "type": "string" 1326 "type": "string"
1266 }, 1327 },
1267 "sdkPipelineOptions": { 1328 "outputInfos": {
1329 "description": "Information about each of the outputs.",
1330 "items": {
1331 "$ref": "SeqMapTaskOutputInfo"
1332 },
1333 "type": "array"
1334 },
1335 "inputs": {
1336 "description": "Information about each of the inputs.",
1337 "items": {
1338 "$ref": "SideInputInfo"
1339 },
1340 "type": "array"
1341 },
1342 "stageName": {
1343 "description": "System-defined name of the stage containing the SeqDo operation.\nUnique across the workflow.",
1344 "type": "string"
1345 },
1346 "systemName": {
1347 "description": "System-defined name of the SeqDo operation.\ nUnique across the workflow.",
1348 "type": "string"
1349 }
1350 },
1351 "type": "object"
1352 },
1353 "WorkerMessageCode": {
1354 "description": "A message code is used to report status and error me ssages to the service.\nThe message codes are intended to be machine readable. T he service will\ntake care of translating these into user understandable message s if\nnecessary.\n\nExample use cases:\n 1. Worker processes reporting successf ul startup.\n 2. Worker processes reporting specific errors (e.g. package stagi ng\n failure).",
1355 "id": "WorkerMessageCode",
1356 "properties": {
1357 "parameters": {
1268 "additionalProperties": { 1358 "additionalProperties": {
1269 "description": "Properties of the object.", 1359 "description": "Properties of the object.",
1270 "type": "any" 1360 "type": "any"
1271 }, 1361 },
1272 "description": "The Cloud Dataflow SDK pipeline options spec ified by the user. These options are passed through the service and are used to recreate the SDK pipeline options on the worker in a language agnostic and platf orm independent way.", 1362 "description": "Parameters contains specific information abo ut the code.\n\nThis is a struct to allow parameters of different types.\n\nExam ples:\n 1. For a \"HARNESS_STARTED\" message parameters might provide the name\n of the worker and additional data like timing information.\n 2. For a \"GCS_ DOWNLOAD_ERROR\" parameters might contain fields listing\n the GCS objects be ing downloaded and fields containing errors.\n\nIn general complex data structur es should be avoided. If a worker\nneeds to send a specific and complicated data structure then please\nconsider defining a new proto and adding it to the data oneof in\nWorkerMessageResponse.\n\nConventions:\n Parameters should only be use d for information that isn't typically passed\n as a label.\n hostname and other worker identifiers should almost always be passed\n as labels since they will b e included on most messages.",
1273 "type": "object" 1363 "type": "object"
1274 }, 1364 },
1275 "internalExperiments": { 1365 "code": {
1276 "additionalProperties": { 1366 "description": "The code is a string intended for consumptio n by a machine that identifies\nthe type of message being sent.\nExamples:\n 1. \"HARNESS_STARTED\" might be used to indicate the worker harness has\n start ed.\n 2. \"GCS_DOWNLOAD_ERROR\" might be used to indicate an error downloading\n a GCS file as part of the boot process of one of the worker containers.\n\nT his is a string and not an enum to make it easy to add new codes without\nwaitin g for an API change.",
1277 "description": "Properties of the object. Contains field @type with type URL.", 1367 "type": "string"
1278 "type": "any" 1368 }
1279 }, 1369 },
1280 "description": "Experimental settings.", 1370 "type": "object"
1281 "type": "object" 1371 },
1282 }, 1372 "CustomSourceLocation": {
1283 "serviceAccountEmail": { 1373 "description": "Identifies the location of a custom souce.",
1284 "description": "Identity to run virtual machines as. Default s to the default account.", 1374 "id": "CustomSourceLocation",
1285 "type": "string" 1375 "properties": {
1286 } 1376 "stateful": {
1287 }, 1377 "description": "Whether this source is stateful.",
1288 "type": "object" 1378 "type": "boolean"
1289 }, 1379 }
1290 "WorkerPool": { 1380 },
1291 "description": "Describes one particular pool of Cloud Dataflow work ers to be instantiated by the Cloud Dataflow service in order to perform the com putations required by a job. Note that a workflow job may use multiple pools, in order to match the various computational requirements of the various stages of the job.", 1381 "type": "object"
1292 "id": "WorkerPool", 1382 },
1293 "properties": { 1383 "MapTask": {
1294 "kind": { 1384 "description": "MapTask consists of an ordered set of instructions, each of which\ndescribes one particular low-level operation for the worker to\np erform in order to accomplish the MapTask's WorkItem.\n\nEach instruction must a ppear in the list before any instructions which\ndepends on its output.",
1295 "description": "The kind of the worker pool; currently only `harness` and `shuffle` are supported.", 1385 "id": "MapTask",
1296 "type": "string" 1386 "properties": {
1297 }, 1387 "systemName": {
1298 "numWorkers": { 1388 "description": "System-defined name of this MapTask.\nUnique across the workflow.",
1299 "description": "Number of Google Compute Engine workers in t his pool needed to execute the job. If zero or unspecified, the service will att empt to choose a reasonable default.", 1389 "type": "string"
1300 "format": "int32", 1390 },
1301 "type": "integer" 1391 "stageName": {
1302 }, 1392 "description": "System-defined name of the stage containing this MapTask.\nUnique across the workflow.",
1303 "packages": { 1393 "type": "string"
1304 "description": "Packages to be installed on workers.", 1394 },
1395 "instructions": {
1396 "description": "The instructions in the MapTask.",
1305 "items": { 1397 "items": {
1306 "$ref": "Package" 1398 "$ref": "ParallelInstruction"
1307 }, 1399 },
1308 "type": "array" 1400 "type": "array"
1309 }, 1401 }
1310 "defaultPackageSet": { 1402 },
1311 "description": "The default package set to install. This all ows the service to select a default set of packages which are useful to worker h arnesses written in a particular language.", 1403 "type": "object"
1312 "enum": [ 1404 },
1313 "DEFAULT_PACKAGE_SET_UNKNOWN", 1405 "FloatingPointMean": {
1314 "DEFAULT_PACKAGE_SET_NONE", 1406 "description": "A representation of a floating point mean metric con tribution.",
1315 "DEFAULT_PACKAGE_SET_JAVA", 1407 "id": "FloatingPointMean",
1316 "DEFAULT_PACKAGE_SET_PYTHON" 1408 "properties": {
1317 ], 1409 "count": {
1318 "type": "string" 1410 "$ref": "SplitInt64",
1319 }, 1411 "description": "The number of values being aggregated."
1320 "machineType": { 1412 },
1321 "description": "Machine type (e.g. \"n1-standard-1\"). If em pty or unspecified, the service will attempt to choose a reasonable default.", 1413 "sum": {
1322 "type": "string" 1414 "description": "The sum of all values being aggregated.",
1323 }, 1415 "format": "double",
1324 "teardownPolicy": { 1416 "type": "number"
1325 "description": "Sets the policy for determining when to turn down worker pool. Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and `TEARDOWN_NEVER`. `TEARDOWN_ALWAYS` means workers are always torn down regar dless of whether the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn down . If the workers are not torn down by the service, they will continue to run and use Google Compute Engine VM resources in the user's project until they are exp licitly terminated by the user. Because of this, Google recommends using the `TE ARDOWN_ALWAYS` policy except for small, manually supervised test jobs. If unknow n or unspecified, the service will attempt to choose a reasonable default.", 1417 }
1326 "enum": [ 1418 },
1327 "TEARDOWN_POLICY_UNKNOWN", 1419 "type": "object"
1328 "TEARDOWN_ALWAYS", 1420 },
1329 "TEARDOWN_ON_SUCCESS", 1421 "ReportWorkItemStatusResponse": {
1330 "TEARDOWN_NEVER" 1422 "description": "Response from a request to report the status of Work Items.",
1331 ], 1423 "id": "ReportWorkItemStatusResponse",
1332 "type": "string" 1424 "properties": {
1333 }, 1425 "workItemServiceStates": {
1334 "diskSizeGb": { 1426 "description": "A set of messages indicating the service-sid e state for each\nWorkItem whose status was reported, in the same order as the\n WorkItemStatus messages in the ReportWorkItemStatusRequest which\nresulting in t his response.",
1335 "description": "Size of root disk for VMs, in GB. If zero or unspecified, the service will attempt to choose a reasonable default.",
1336 "format": "int32",
1337 "type": "integer"
1338 },
1339 "diskType": {
1340 "description": "Type of root disk for VMs. If empty or unspe cified, the service will attempt to choose a reasonable default.",
1341 "type": "string"
1342 },
1343 "diskSourceImage": {
1344 "description": "Fully qualified source image for disks.",
1345 "type": "string"
1346 },
1347 "zone": {
1348 "description": "Zone to run the worker pools in. If empty or unspecified, the service will attempt to choose a reasonable default.",
1349 "type": "string"
1350 },
1351 "taskrunnerSettings": {
1352 "$ref": "TaskRunnerSettings",
1353 "description": "Settings passed through to Google Compute En gine workers when using the standard Dataflow task runner. Users should ignore t his field."
1354 },
1355 "onHostMaintenance": {
1356 "description": "The action to take on host maintenance, as d efined by the Google Compute Engine API.",
1357 "type": "string"
1358 },
1359 "dataDisks": {
1360 "description": "Data disks that are used by a VM in this wor kflow.",
1361 "items": { 1427 "items": {
1362 "$ref": "Disk" 1428 "$ref": "WorkItemServiceState"
1363 }, 1429 },
1364 "type": "array" 1430 "type": "array"
1365 }, 1431 }
1366 "metadata": { 1432 },
1367 "additionalProperties": { 1433 "type": "object"
1368 "type": "string" 1434 },
1369 }, 1435 "InstructionOutput": {
1370 "description": "Metadata to set on the Google Compute Engine VMs.", 1436 "description": "An output of an instruction.",
1371 "type": "object" 1437 "id": "InstructionOutput",
1372 },
1373 "autoscalingSettings": {
1374 "$ref": "AutoscalingSettings",
1375 "description": "Settings for autoscaling of this WorkerPool. "
1376 },
1377 "poolArgs": {
1378 "additionalProperties": {
1379 "description": "Properties of the object. Contains field @type with type URL.",
1380 "type": "any"
1381 },
1382 "description": "Extra arguments for this worker pool.",
1383 "type": "object"
1384 },
1385 "network": {
1386 "description": "Network to which VMs will be assigned. If em pty or unspecified, the service will use the network \"default\".",
1387 "type": "string"
1388 },
1389 "subnetwork": {
1390 "description": "Subnetwork to which VMs will be assigned, if desired. Expected to be of the form \"regions/REGION/subnetworks/SUBNETWORK\"." ,
1391 "type": "string"
1392 },
1393 "workerHarnessContainerImage": {
1394 "description": "Required. Docker container image that execut es the Cloud Dataflow worker harness, residing in Google Container Registry.",
1395 "type": "string"
1396 },
1397 "numThreadsPerWorker": {
1398 "description": "The number of threads per worker harness. If empty or unspecified, the service will choose a number of threads (according to the number of cores on the selected machine type for batch, or 1 by convention for streaming).",
1399 "format": "int32",
1400 "type": "integer"
1401 },
1402 "ipConfiguration": {
1403 "description": "Configuration for VM IPs.",
1404 "enum": [
1405 "WORKER_IP_UNSPECIFIED",
1406 "WORKER_IP_PUBLIC",
1407 "WORKER_IP_PRIVATE"
1408 ],
1409 "type": "string"
1410 }
1411 },
1412 "type": "object"
1413 },
1414 "Package": {
1415 "description": "The packages that must be installed in order for a w orker to run the steps of the Cloud Dataflow job that will be assigned to its wo rker pool. This is the mechanism by which the Cloud Dataflow SDK causes code to be loaded onto the workers. For example, the Cloud Dataflow Java SDK might use t his to install jars containing the user's code and all of the various dependenci es (libraries, data files, etc.) required in order for that code to run.",
1416 "id": "Package",
1417 "properties": { 1438 "properties": {
1418 "name": { 1439 "name": {
1419 "description": "The name of the package.", 1440 "description": "The user-provided name of this output.",
1420 "type": "string" 1441 "type": "string"
1421 }, 1442 },
1422 "location": { 1443 "originalName": {
1423 "description": "The resource to read the package from. The s upported resource type is: Google Cloud Storage: storage.googleapis.com/{bucket} bucket.storage.googleapis.com/", 1444 "description": "System-defined name for this output in the o riginal workflow graph.\nOutputs that do not contribute to an original instructi on do not set this.",
1424 "type": "string" 1445 "type": "string"
1425 } 1446 },
1426 }, 1447 "systemName": {
1427 "type": "object" 1448 "description": "System-defined name of this output.\nUnique across the workflow.",
1428 }, 1449 "type": "string"
1429 "TaskRunnerSettings": { 1450 },
1430 "description": "Taskrunner configuration settings.", 1451 "onlyCountKeyBytes": {
1431 "id": "TaskRunnerSettings", 1452 "description": "For system-generated byte and mean byte metr ics, certain instructions\nshould only report the key size.",
1432 "properties": {
1433 "taskUser": {
1434 "description": "The UNIX user ID on the worker VM to use for tasks launched by taskrunner; e.g. \"root\".",
1435 "type": "string"
1436 },
1437 "taskGroup": {
1438 "description": "The UNIX group ID on the worker VM to use fo r tasks launched by taskrunner; e.g. \"wheel\".",
1439 "type": "string"
1440 },
1441 "oauthScopes": {
1442 "description": "The OAuth2 scopes to be requested by the tas krunner in order to access the Cloud Dataflow API.",
1443 "items": {
1444 "type": "string"
1445 },
1446 "type": "array"
1447 },
1448 "baseUrl": {
1449 "description": "The base URL for the taskrunner to use when accessing Google Cloud APIs. When workers access Google Cloud APIs, they logical ly do so via relative URLs. If this field is specified, it supplies the base URL to use for resolving these relative URLs. The normative algorithm used is defin ed by RFC 1808, \"Relative Uniform Resource Locators\". If not specified, the de fault value is \"http://www.googleapis.com/\"",
1450 "type": "string"
1451 },
1452 "dataflowApiVersion": {
1453 "description": "The API version of endpoint, e.g. \"v1b3\"",
1454 "type": "string"
1455 },
1456 "parallelWorkerSettings": {
1457 "$ref": "WorkerSettings",
1458 "description": "The settings to pass to the parallel worker harness."
1459 },
1460 "baseTaskDir": {
1461 "description": "The location on the worker for task-specific subdirectories.",
1462 "type": "string"
1463 },
1464 "continueOnException": {
1465 "description": "Whether to continue taskrunner if an excepti on is hit.",
1466 "type": "boolean" 1453 "type": "boolean"
1467 }, 1454 },
1468 "logToSerialconsole": { 1455 "onlyCountValueBytes": {
1469 "description": "Whether to send taskrunner log info to Googl e Compute Engine VM serial console.", 1456 "description": "For system-generated byte and mean byte metr ics, certain instructions\nshould only report the value size.",
1470 "type": "boolean" 1457 "type": "boolean"
1471 }, 1458 },
1472 "alsologtostderr": { 1459 "codec": {
1473 "description": "Whether to also send taskrunner log info to stderr.",
1474 "type": "boolean"
1475 },
1476 "logUploadLocation": {
1477 "description": "Indicates where to put logs. If this is not specified, the logs will not be uploaded. The supported resource type is: Google Cloud Storage: storage.googleapis.com/{bucket}/{object} bucket.storage.googleap is.com/{object}",
1478 "type": "string"
1479 },
1480 "logDir": {
1481 "description": "The directory on the VM to store logs.",
1482 "type": "string"
1483 },
1484 "tempStoragePrefix": {
1485 "description": "The prefix of the resources the taskrunner s hould use for temporary storage. The supported resource type is: Google Cloud St orage: storage.googleapis.com/{bucket}/{object} bucket.storage.googleapis.com/{o bject}",
1486 "type": "string"
1487 },
1488 "harnessCommand": {
1489 "description": "The command to launch the worker harness.",
1490 "type": "string"
1491 },
1492 "workflowFileName": {
1493 "description": "The file to store the workflow in.",
1494 "type": "string"
1495 },
1496 "commandlinesFileName": {
1497 "description": "The file to store preprocessing commands in. ",
1498 "type": "string"
1499 },
1500 "vmId": {
1501 "description": "The ID string of the VM.",
1502 "type": "string"
1503 },
1504 "languageHint": {
1505 "description": "The suggested backend language.",
1506 "type": "string"
1507 },
1508 "streamingWorkerMainClass": {
1509 "description": "The streaming worker main class name.",
1510 "type": "string"
1511 }
1512 },
1513 "type": "object"
1514 },
1515 "WorkerSettings": {
1516 "description": "Provides data to pass through to the worker harness. ",
1517 "id": "WorkerSettings",
1518 "properties": {
1519 "baseUrl": {
1520 "description": "The base URL for accessing Google Cloud APIs . When workers access Google Cloud APIs, they logically do so via relative URLs. If this field is specified, it supplies the base URL to use for resolving these relative URLs. The normative algorithm used is defined by RFC 1808, \"Relative Uniform Resource Locators\". If not specified, the default value is \"http://www .googleapis.com/\"",
1521 "type": "string"
1522 },
1523 "reportingEnabled": {
1524 "description": "Whether to send work progress updates to the service.",
1525 "type": "boolean"
1526 },
1527 "servicePath": {
1528 "description": "The Cloud Dataflow service path relative to the root URL, for example, \"dataflow/v1b3/projects\".",
1529 "type": "string"
1530 },
1531 "shuffleServicePath": {
1532 "description": "The Shuffle service path relative to the roo t URL, for example, \"shuffle/v1beta1\".",
1533 "type": "string"
1534 },
1535 "workerId": {
1536 "description": "The ID of the worker running this pipeline." ,
1537 "type": "string"
1538 },
1539 "tempStoragePrefix": {
1540 "description": "The prefix of the resources the system shoul d use for temporary storage. The supported resource type is: Google Cloud Storag e: storage.googleapis.com/{bucket}/{object} bucket.storage.googleapis.com/{objec t}",
1541 "type": "string"
1542 }
1543 },
1544 "type": "object"
1545 },
1546 "Disk": {
1547 "description": "Describes the data disk used by a workflow job.",
1548 "id": "Disk",
1549 "properties": {
1550 "sizeGb": {
1551 "description": "Size of disk in GB. If zero or unspecified, the service will attempt to choose a reasonable default.",
1552 "format": "int32",
1553 "type": "integer"
1554 },
1555 "diskType": {
1556 "description": "Disk storage type, as defined by Google Comp ute Engine. This must be a disk type appropriate to the project and zone in whic h the workers will run. If unknown or unspecified, the service will attempt to c hoose a reasonable default. For example, the standard persistent disk type is a resource name typically ending in \"pd-standard\". If SSD persistent disks are a vailable, the resource name typically ends with \"pd-ssd\". The actual valid val ues are defined the Google Compute Engine API, not by the Cloud Dataflow API; co nsult the Google Compute Engine documentation for more information about determi ning the set of available disk types for a particular project and zone. Google C ompute Engine Disk types are local to a particular project in a particular zone, and so the resource name will typically look something like this: compute.googl eapis.com/projects/project-id/zones/zone/diskTypes/pd-standard",
1557 "type": "string"
1558 },
1559 "mountPoint": {
1560 "description": "Directory in a VM where disk is mounted.",
1561 "type": "string"
1562 }
1563 },
1564 "type": "object"
1565 },
1566 "AutoscalingSettings": {
1567 "description": "Settings for WorkerPool autoscaling.",
1568 "id": "AutoscalingSettings",
1569 "properties": {
1570 "algorithm": {
1571 "description": "The algorithm to use for autoscaling.",
1572 "enum": [
1573 "AUTOSCALING_ALGORITHM_UNKNOWN",
1574 "AUTOSCALING_ALGORITHM_NONE",
1575 "AUTOSCALING_ALGORITHM_BASIC"
1576 ],
1577 "type": "string"
1578 },
1579 "maxNumWorkers": {
1580 "description": "The maximum number of workers to cap scaling at.",
1581 "format": "int32",
1582 "type": "integer"
1583 }
1584 },
1585 "type": "object"
1586 },
1587 "Step": {
1588 "description": "Defines a particular step within a Cloud Dataflow jo b. A job consists of multiple steps, each of which performs some specific operat ion as part of the overall job. Data is typically passed from one step to anothe r as part of the job. Here's an example of a sequence of steps which together im plement a Map-Reduce job: * Read a collection of data from some source, parsing the collection's elements. * Validate the elements. * Apply a user-defined funct ion to map each element to some value and extract an element-specific key value. * Group elements with the same key into a single element with that key, transfo rming a multiply-keyed collection into a uniquely-keyed collection. * Write the elements out to some data sink. Note that the Cloud Dataflow service may be used to run many different types of jobs, not just Map-Reduce.",
1589 "id": "Step",
1590 "properties": {
1591 "kind": {
1592 "description": "The kind of step in the Cloud Dataflow job." ,
1593 "type": "string"
1594 },
1595 "name": {
1596 "description": "The name that identifies the step. This must be unique for each step with respect to all other steps in the Cloud Dataflow j ob.",
1597 "type": "string"
1598 },
1599 "properties": {
1600 "additionalProperties": { 1460 "additionalProperties": {
1601 "description": "Properties of the object.", 1461 "description": "Properties of the object.",
1602 "type": "any" 1462 "type": "any"
1603 }, 1463 },
1604 "description": "Named properties associated with the step. E ach kind of predefined step has its own required set of properties.", 1464 "description": "The codec to use to encode data being writte n via this output.",
1605 "type": "object" 1465 "type": "object"
1606 } 1466 }
1607 }, 1467 },
1608 "type": "object" 1468 "type": "object"
1609 }, 1469 },
1610 "JobExecutionInfo": { 1470 "CreateJobFromTemplateRequest": {
1611 "description": "Additional information about how a Cloud Dataflow jo b will be executed that isn't contained in the submitted job.", 1471 "description": "A request to create a Cloud Dataflow job from a temp late.",
1612 "id": "JobExecutionInfo", 1472 "id": "CreateJobFromTemplateRequest",
1613 "properties": { 1473 "properties": {
1614 "stages": { 1474 "gcsPath": {
1475 "description": "Required. A Cloud Storage path to the templa te from which to\ncreate the job.\nMust be a valid Cloud Storage URL, beginning with `gs://`.",
1476 "type": "string"
1477 },
1478 "environment": {
1479 "$ref": "RuntimeEnvironment",
1480 "description": "The runtime environment for the job."
1481 },
1482 "parameters": {
1615 "additionalProperties": { 1483 "additionalProperties": {
1616 "$ref": "JobExecutionStageInfo" 1484 "type": "string"
1617 }, 1485 },
1618 "description": "A mapping from each stage to the information about that stage.", 1486 "description": "The runtime parameters to pass to the job.",
1619 "type": "object" 1487 "type": "object"
1620 } 1488 },
1621 }, 1489 "jobName": {
1622 "type": "object" 1490 "description": "Required. The job name to use for the create d job.",
1623 }, 1491 "type": "string"
1624 "JobExecutionStageInfo": { 1492 }
1625 "description": "Contains information about how a particular google.d ataflow.v1beta3.Step will be executed.", 1493 },
1626 "id": "JobExecutionStageInfo", 1494 "type": "object"
1627 "properties": { 1495 },
1628 "stepName": { 1496 "IntegerMean": {
1629 "description": "The steps associated with the execution stag e. Note that stages may have several steps, and that a given step might be run b y more than one stage.", 1497 "description": "A representation of an integer mean metric contribut ion.",
1630 "items": { 1498 "id": "IntegerMean",
1631 "type": "string" 1499 "properties": {
1632 }, 1500 "sum": {
1633 "type": "array" 1501 "$ref": "SplitInt64",
1502 "description": "The sum of all values being aggregated."
1503 },
1504 "count": {
1505 "$ref": "SplitInt64",
1506 "description": "The number of values being aggregated."
1634 } 1507 }
1635 }, 1508 },
1636 "type": "object" 1509 "type": "object"
1637 }, 1510 },
1638 "ListJobsResponse": { 1511 "ListJobsResponse": {
1639 "description": "Response to a request to list Cloud Dataflow jobs. T his may be a partial response, depending on the page size in the ListJobsRequest .", 1512 "description": "Response to a request to list Cloud Dataflow jobs. This may be a partial\nresponse, depending on the page size in the ListJobsReque st.",
1640 "id": "ListJobsResponse", 1513 "id": "ListJobsResponse",
1641 "properties": { 1514 "properties": {
1642 "jobs": {
1643 "description": "A subset of the requested job information.",
1644 "items": {
1645 "$ref": "Job"
1646 },
1647 "type": "array"
1648 },
1649 "nextPageToken": { 1515 "nextPageToken": {
1650 "description": "Set if there may be more results than fit in this response.", 1516 "description": "Set if there may be more results than fit in this response.",
1651 "type": "string" 1517 "type": "string"
1652 }, 1518 },
1653 "failedLocation": { 1519 "failedLocation": {
1654 "description": "Zero or more messages describing locations t hat failed to respond.", 1520 "description": "Zero or more messages describing locations t hat failed to respond.",
1655 "items": { 1521 "items": {
1656 "$ref": "FailedLocation" 1522 "$ref": "FailedLocation"
1657 }, 1523 },
1658 "type": "array" 1524 "type": "array"
1659 } 1525 },
1660 }, 1526 "jobs": {
1661 "type": "object" 1527 "description": "A subset of the requested job information.",
1662 }, 1528 "items": {
1663 "FailedLocation": { 1529 "$ref": "Job"
1664 "description": "Indicates which location failed to respond to a requ est for data.", 1530 },
1665 "id": "FailedLocation", 1531 "type": "array"
1666 "properties": { 1532 }
1667 "name": { 1533 },
1668 "description": "The name of the failed location.", 1534 "type": "object"
1669 "type": "string" 1535 },
1670 } 1536 "ComputationTopology": {
1671 }, 1537 "description": "All configuration data for a particular Computation. ",
1672 "type": "object" 1538 "id": "ComputationTopology",
1673 }, 1539 "properties": {
1674 "ListJobMessagesResponse": { 1540 "userStageName": {
1675 "description": "Response to a request to list job messages.", 1541 "description": "The user stage name.",
1676 "id": "ListJobMessagesResponse", 1542 "type": "string"
1677 "properties": { 1543 },
1678 "jobMessages": { 1544 "outputs": {
1679 "description": "Messages in ascending timestamp order.", 1545 "description": "The outputs from the computation.",
1680 "items": { 1546 "items": {
1681 "$ref": "JobMessage" 1547 "$ref": "StreamLocation"
1682 }, 1548 },
1683 "type": "array" 1549 "type": "array"
1684 }, 1550 },
1685 "nextPageToken": { 1551 "stateFamilies": {
1686 "description": "The token to obtain the next page of results if there are more.", 1552 "description": "The state family values.",
1687 "type": "string" 1553 "items": {
1688 } 1554 "$ref": "StateFamilyConfig"
1689 }, 1555 },
1690 "type": "object" 1556 "type": "array"
1691 }, 1557 },
1692 "JobMessage": { 1558 "systemStageName": {
1693 "description": "A particular message pertaining to a Dataflow job.", 1559 "description": "The system stage name.",
1694 "id": "JobMessage", 1560 "type": "string"
1695 "properties": { 1561 },
1696 "id": { 1562 "computationId": {
1697 "description": "Identifies the message. This is automaticall y generated by the service; the caller should treat it as an opaque string.", 1563 "description": "The ID of the computation.",
1698 "type": "string" 1564 "type": "string"
1699 }, 1565 },
1700 "time": { 1566 "inputs": {
1701 "description": "The timestamp of the message.", 1567 "description": "The inputs to the computation.",
1702 "type": "string" 1568 "items": {
1703 }, 1569 "$ref": "StreamLocation"
1704 "messageText": { 1570 },
1705 "description": "The text of the message.", 1571 "type": "array"
1706 "type": "string" 1572 },
1707 }, 1573 "keyRanges": {
1708 "messageImportance": { 1574 "description": "The key ranges processed by the computation. ",
1709 "description": "Importance level of the message.", 1575 "items": {
1710 "enum": [ 1576 "$ref": "KeyRangeLocation"
1711 "JOB_MESSAGE_IMPORTANCE_UNKNOWN", 1577 },
1712 "JOB_MESSAGE_DEBUG", 1578 "type": "array"
1713 "JOB_MESSAGE_DETAILED",
1714 "JOB_MESSAGE_BASIC",
1715 "JOB_MESSAGE_WARNING",
1716 "JOB_MESSAGE_ERROR"
1717 ],
1718 "type": "string"
1719 }
1720 },
1721 "type": "object"
1722 },
1723 "JobMetrics": {
1724 "description": "JobMetrics contains a collection of metrics descibin g the detailed progress of a Dataflow job. Metrics correspond to user-defined an d system-defined metrics in the job. This resource captures only the most recent values of each metric; time-series data can be queried for them (under the same metric names) from Cloud Monitoring.",
1725 "id": "JobMetrics",
1726 "properties": {
1727 "metricTime": {
1728 "description": "Timestamp as of which metric values are curr ent.",
1729 "type": "string"
1730 },
1731 "metrics": {
1732 "description": "All metrics for this job.",
1733 "items": {
1734 "$ref": "MetricUpdate"
1735 },
1736 "type": "array"
1737 }
1738 },
1739 "type": "object"
1740 },
1741 "MetricUpdate": {
1742 "description": "Describes the state of a metric.",
1743 "id": "MetricUpdate",
1744 "properties": {
1745 "name": {
1746 "$ref": "MetricStructuredName",
1747 "description": "Name of the metric."
1748 },
1749 "kind": {
1750 "description": "Metric aggregation kind. The possible metric aggregation kinds are \"Sum\", \"Max\", \"Min\", \"Mean\", \"Set\", \"And\", an d \"Or\". The specified aggregation kind is case-insensitive. If omitted, this i s not an aggregated value but instead a single metric sample value.",
1751 "type": "string"
1752 },
1753 "cumulative": {
1754 "description": "True if this metric is reported as the total cumulative aggregate value accumulated since the worker started working on this WorkItem. By default this is false, indicating that this metric is reported as a delta that is not associated with any WorkItem.",
1755 "type": "boolean"
1756 },
1757 "scalar": {
1758 "description": "Worker-computed aggregate value for aggregat ion kinds \"Sum\", \"Max\", \"Min\", \"And\", and \"Or\". The possible value typ es are Long, Double, and Boolean.",
1759 "type": "any"
1760 },
1761 "meanSum": {
1762 "description": "Worker-computed aggregate value for the \"Me an\" aggregation kind. This holds the sum of the aggregated values and is used i n combination with mean_count below to obtain the actual mean aggregate value. T he only possible value types are Long and Double.",
1763 "type": "any"
1764 },
1765 "meanCount": {
1766 "description": "Worker-computed aggregate value for the \"Me an\" aggregation kind. This holds the count of the aggregated values and is used in combination with mean_sum above to obtain the actual mean aggregate value. T he only possible value type is Long.",
1767 "type": "any"
1768 },
1769 "set": {
1770 "description": "Worker-computed aggregate value for the \"Se t\" aggregation kind. The only possible value type is a list of Values whose typ e can be Long, Double, or String, according to the metric's type. All Values in the list must be of the same type.",
1771 "type": "any"
1772 },
1773 "internal": {
1774 "description": "Worker-computed aggregate value for internal use by the Dataflow service.",
1775 "type": "any"
1776 },
1777 "updateTime": {
1778 "description": "Timestamp associated with the metric value. Optional when workers are reporting work progress; it will be filled in response s from the metrics API.",
1779 "type": "string"
1780 }
1781 },
1782 "type": "object"
1783 },
1784 "MetricStructuredName": {
1785 "description": "Identifies a metric, by describing the source which generated the metric.",
1786 "id": "MetricStructuredName",
1787 "properties": {
1788 "origin": {
1789 "description": "Origin (namespace) of metric name. May be bl ank for user-define metrics; will be \"dataflow\" for metrics defined by the Dat aflow service or SDK.",
1790 "type": "string"
1791 },
1792 "name": {
1793 "description": "Worker-defined metric name.",
1794 "type": "string"
1795 },
1796 "context": {
1797 "additionalProperties": {
1798 "type": "string"
1799 },
1800 "description": "Zero or more labeled fields which identify t he part of the job this metric is associated with, such as the name of a step or collection. For example, built-in counters associated with steps will have cont ext['step'] = . Counters associated with PCollections in the SDK will have conte xt['pcollection'] =\n.",
1801 "type": "object"
1802 }
1803 },
1804 "type": "object"
1805 },
1806 "CreateJobFromTemplateRequest": {
1807 "description": "A request to create a Cloud Dataflow job from a temp late.",
1808 "id": "CreateJobFromTemplateRequest",
1809 "properties": {
1810 "jobName": {
1811 "description": "Required. The job name to use for the create d job.",
1812 "type": "string"
1813 },
1814 "gcsPath": {
1815 "description": "Required. A Cloud Storage path to the templa te from which to create the job. Must be a valid Cloud Storage URL, beginning wi th `gs://`.",
1816 "type": "string"
1817 },
1818 "parameters": {
1819 "additionalProperties": {
1820 "type": "string"
1821 },
1822 "description": "The runtime parameters to pass to the job.",
1823 "type": "object"
1824 },
1825 "environment": {
1826 "$ref": "RuntimeEnvironment",
1827 "description": "The runtime environment for the job."
1828 } 1579 }
1829 }, 1580 },
1830 "type": "object" 1581 "type": "object"
1831 }, 1582 },
1832 "RuntimeEnvironment": { 1583 "RuntimeEnvironment": {
1833 "description": "The environment values to set at runtime.", 1584 "description": "The environment values to set at runtime.",
1834 "id": "RuntimeEnvironment", 1585 "id": "RuntimeEnvironment",
1835 "properties": { 1586 "properties": {
1587 "serviceAccountEmail": {
1588 "description": "The email address of the service account to run the job as.",
1589 "type": "string"
1590 },
1591 "tempLocation": {
1592 "description": "The Cloud Storage path to use for temporary files.\nMust be a valid Cloud Storage URL, beginning with `gs://`.",
1593 "type": "string"
1594 },
1595 "bypassTempDirValidation": {
1596 "description": "Whether to bypass the safety checks for the job's temporary directory.\nUse with caution.",
1597 "type": "boolean"
1598 },
1836 "maxWorkers": { 1599 "maxWorkers": {
1837 "description": "The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000 .", 1600 "description": "The maximum number of Google Compute Engine instances to be made\navailable to your pipeline during execution, from 1 to 100 0.",
1838 "format": "int32", 1601 "format": "int32",
1839 "type": "integer" 1602 "type": "integer"
1840 }, 1603 },
1841 "zone": { 1604 "zone": {
1842 "description": "The Compute Engine [availability zone](https ://cloud.google.com/compute/docs/regions-zones/regions-zones) for launching work er instances to run your pipeline.", 1605 "description": "The Compute Engine [availability zone](https ://cloud.google.com/compute/docs/regions-zones/regions-zones)\nfor launching wor ker instances to run your pipeline.",
1843 "type": "string" 1606 "type": "string"
1844 }, 1607 }
1845 "serviceAccountEmail": { 1608 },
1846 "description": "The email address of the service account to run the job as.", 1609 "type": "object"
1847 "type": "string" 1610 },
1848 }, 1611 "MountedDataDisk": {
1849 "tempLocation": { 1612 "description": "Describes mounted data disk.",
1850 "description": "The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with `gs://`.", 1613 "id": "MountedDataDisk",
1851 "type": "string" 1614 "properties": {
1852 }, 1615 "dataDisk": {
1853 "bypassTempDirValidation": { 1616 "description": "The name of the data disk.\nThis name is loc al to the Google Cloud Platform project and uniquely\nidentifies the disk within that project, for example\n\"myproject-1014-104817-4c2-harness-0-disk-1\".",
1854 "description": "Whether to bypass the safety checks for the job's temporary directory. Use with caution.", 1617 "type": "string"
1855 "type": "boolean" 1618 }
1856 } 1619 },
1857 }, 1620 "type": "object"
1858 "type": "object" 1621 },
1859 }, 1622 "StreamingSideInputLocation": {
1860 "ReportWorkItemStatusRequest": { 1623 "description": "Identifies the location of a streaming side input.",
1861 "description": "Request to report the status of WorkItems.", 1624 "id": "StreamingSideInputLocation",
1862 "id": "ReportWorkItemStatusRequest", 1625 "properties": {
1863 "properties": { 1626 "stateFamily": {
1864 "workerId": { 1627 "description": "Identifies the state family where this side input is stored.",
1865 "description": "The ID of the worker reporting the WorkItem status. If this does not match the ID of the worker which the Dataflow service b elieves currently has the lease on the WorkItem, the report will be dropped (wit h an error response).", 1628 "type": "string"
1866 "type": "string" 1629 },
1867 }, 1630 "tag": {
1868 "workItemStatuses": { 1631 "description": "Identifies the particular side input within the streaming Dataflow job.",
1869 "description": "The order is unimportant, except that the or der of the WorkItemServiceState messages in the ReportWorkItemStatusResponse cor responds to the order of WorkItemStatus messages here.", 1632 "type": "string"
1870 "items": { 1633 }
1871 "$ref": "WorkItemStatus" 1634 },
1872 }, 1635 "type": "object"
1873 "type": "array" 1636 },
1874 }, 1637 "Job": {
1875 "currentWorkerTime": { 1638 "description": "Defines a job to be run by the Cloud Dataflow servic e.",
1876 "description": "The current timestamp at the worker.", 1639 "id": "Job",
1640 "properties": {
1641 "requestedState": {
1642 "description": "The job's requested state.\n\n`UpdateJob` ma y be used to switch between the `JOB_STATE_STOPPED` and\n`JOB_STATE_RUNNING` sta tes, by setting requested_state. `UpdateJob` may\nalso be used to directly set a job's requested state to\n`JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocab ly terminating the\njob if it has not already reached a terminal state.",
1643 "enum": [
1644 "JOB_STATE_UNKNOWN",
1645 "JOB_STATE_STOPPED",
1646 "JOB_STATE_RUNNING",
1647 "JOB_STATE_DONE",
1648 "JOB_STATE_FAILED",
1649 "JOB_STATE_CANCELLED",
1650 "JOB_STATE_UPDATED",
1651 "JOB_STATE_DRAINING",
1652 "JOB_STATE_DRAINED"
1653 ],
1654 "enumDescriptions": [
1655 "The job's run state isn't specified.",
1656 "`JOB_STATE_STOPPED` indicates that the job has not\nyet started to run.",
1657 "`JOB_STATE_RUNNING` indicates that the job is currently running.",
1658 "`JOB_STATE_DONE` indicates that the job has successfull y completed.\nThis is a terminal job state. This state may be set by the Cloud Dataflow\nservice, as a transition from `JOB_STATE_RUNNING`. It may also be set via a\nCloud Dataflow `UpdateJob` call, if the job has not yet reached a termina l\nstate.",
1659 "`JOB_STATE_FAILED` indicates that the job has failed. This is a\nterminal job state. This state may only be set by the Cloud Dataflow \nservice, and only as a transition from `JOB_STATE_RUNNING`.",
1660 "`JOB_STATE_CANCELLED` indicates that the job has been e xplicitly\ncancelled. This is a terminal job state. This state may only be\nset via a Cloud Dataflow `UpdateJob` call, and only if the job has not\nyet reached another terminal state.",
1661 "`JOB_STATE_UPDATED` indicates that the job was successf ully updated,\nmeaning that this job was stopped and another job was started, in heriting\nstate from this one. This is a terminal job state. This state may only be\nset by the Cloud Dataflow service, and only as a transition from\n`JOB_STAT E_RUNNING`.",
1662 "`JOB_STATE_DRAINING` indicates that the job is in the p rocess of draining.\nA draining job has stopped pulling from its input sources a nd is processing\nany data that remains in-flight. This state may be set via a C loud Dataflow\n`UpdateJob` call, but only as a transition from `JOB_STATE_RUNNIN G`. Jobs\nthat are draining may only transition to `JOB_STATE_DRAINED`,\n`JOB_ST ATE_CANCELLED`, or `JOB_STATE_FAILED`.",
1663 "`JOB_STATE_DRAINED` indicates that the job has been dra ined.\nA drained job terminated by stopping pulling from its input sources and\n processing any data that remained in-flight when draining was requested.\nThis s tate is a terminal state, may only be set by the Cloud Dataflow\nservice, and on ly as a transition from `JOB_STATE_DRAINING`."
1664 ],
1665 "type": "string"
1666 },
1667 "tempFiles": {
1668 "description": "A set of files the system should be aware of that are used\nfor temporary storage. These temporary files will be\nremoved on job completion.\nNo duplicates are allowed.\nNo file patterns are supported.\n\ nThe supported files are:\n\nGoogle Cloud Storage:\n\n storage.googleapis.com/ {bucket}/{object}\n bucket.storage.googleapis.com/{object}",
1669 "items": {
1670 "type": "string"
1671 },
1672 "type": "array"
1673 },
1674 "clientRequestId": {
1675 "description": "The client's unique identifier of the job, r e-used across retried attempts.\nIf this field is set, the service will ensure i ts uniqueness.\nThe request to create a job will fail if the service has knowled ge of a\npreviously submitted job with the same client's ID and job name.\nThe c aller may use this field to ensure idempotence of job\ncreation across retried a ttempts to create a job.\nBy default, the field is empty and, in that case, the service ignores it.",
1676 "type": "string"
1677 },
1678 "name": {
1679 "description": "The user-specified Cloud Dataflow job name.\ n\nOnly one Job with a given name may exist in a project at any\ngiven time. If a caller attempts to create a Job with the same\nname as an already-existing Job , the attempt returns the\nexisting Job.\n\nThe name must match the regular expr ession\n`[a-z]([-a-z0-9]{0,38}[a-z0-9])?`",
1680 "type": "string"
1681 },
1682 "steps": {
1683 "description": "The top-level steps that constitute the enti re job.",
1684 "items": {
1685 "$ref": "Step"
1686 },
1687 "type": "array"
1688 },
1689 "replacedByJobId": {
1690 "description": "If another job is an update of this job (and thus, this job is in\n`JOB_STATE_UPDATED`), this field contains the ID of that job.",
1691 "type": "string"
1692 },
1693 "executionInfo": {
1694 "$ref": "JobExecutionInfo",
1695 "description": "Information about how the Cloud Dataflow ser vice will run the job."
1696 },
1697 "id": {
1698 "description": "The unique ID of this job.\n\nThis field is set by the Cloud Dataflow service when the Job is\ncreated, and is immutable for the life of the job.",
1699 "type": "string"
1700 },
1701 "currentState": {
1702 "description": "The current state of the job.\n\nJobs are cr eated in the `JOB_STATE_STOPPED` state unless otherwise\nspecified.\n\nA job in the `JOB_STATE_RUNNING` state may asynchronously enter a\nterminal state. After a job has reached a terminal state, no\nfurther state updates may be made.\n\nTh is field may be mutated by the Cloud Dataflow service;\ncallers cannot mutate it .",
1703 "enum": [
1704 "JOB_STATE_UNKNOWN",
1705 "JOB_STATE_STOPPED",
1706 "JOB_STATE_RUNNING",
1707 "JOB_STATE_DONE",
1708 "JOB_STATE_FAILED",
1709 "JOB_STATE_CANCELLED",
1710 "JOB_STATE_UPDATED",
1711 "JOB_STATE_DRAINING",
1712 "JOB_STATE_DRAINED"
1713 ],
1714 "enumDescriptions": [
1715 "The job's run state isn't specified.",
1716 "`JOB_STATE_STOPPED` indicates that the job has not\nyet started to run.",
1717 "`JOB_STATE_RUNNING` indicates that the job is currently running.",
1718 "`JOB_STATE_DONE` indicates that the job has successfull y completed.\nThis is a terminal job state. This state may be set by the Cloud Dataflow\nservice, as a transition from `JOB_STATE_RUNNING`. It may also be set via a\nCloud Dataflow `UpdateJob` call, if the job has not yet reached a termina l\nstate.",
1719 "`JOB_STATE_FAILED` indicates that the job has failed. This is a\nterminal job state. This state may only be set by the Cloud Dataflow \nservice, and only as a transition from `JOB_STATE_RUNNING`.",
1720 "`JOB_STATE_CANCELLED` indicates that the job has been e xplicitly\ncancelled. This is a terminal job state. This state may only be\nset via a Cloud Dataflow `UpdateJob` call, and only if the job has not\nyet reached another terminal state.",
1721 "`JOB_STATE_UPDATED` indicates that the job was successf ully updated,\nmeaning that this job was stopped and another job was started, in heriting\nstate from this one. This is a terminal job state. This state may only be\nset by the Cloud Dataflow service, and only as a transition from\n`JOB_STAT E_RUNNING`.",
1722 "`JOB_STATE_DRAINING` indicates that the job is in the p rocess of draining.\nA draining job has stopped pulling from its input sources a nd is processing\nany data that remains in-flight. This state may be set via a C loud Dataflow\n`UpdateJob` call, but only as a transition from `JOB_STATE_RUNNIN G`. Jobs\nthat are draining may only transition to `JOB_STATE_DRAINED`,\n`JOB_ST ATE_CANCELLED`, or `JOB_STATE_FAILED`.",
1723 "`JOB_STATE_DRAINED` indicates that the job has been dra ined.\nA drained job terminated by stopping pulling from its input sources and\n processing any data that remained in-flight when draining was requested.\nThis s tate is a terminal state, may only be set by the Cloud Dataflow\nservice, and on ly as a transition from `JOB_STATE_DRAINING`."
1724 ],
1877 "type": "string" 1725 "type": "string"
1878 }, 1726 },
1879 "location": { 1727 "location": {
1880 "description": "The location which contains the WorkItem's j ob.", 1728 "description": "The location that contains this job.",
1881 "type": "string" 1729 "type": "string"
1882 } 1730 },
1883 }, 1731 "currentStateTime": {
1884 "type": "object" 1732 "description": "The timestamp associated with the current st ate.",
1885 }, 1733 "format": "google-datetime",
1886 "WorkItemStatus": { 1734 "type": "string"
1887 "description": "Conveys a worker's progress through the work describ ed by a WorkItem.", 1735 },
1888 "id": "WorkItemStatus", 1736 "transformNameMapping": {
1889 "properties": { 1737 "additionalProperties": {
1890 "workItemId": { 1738 "type": "string"
1891 "description": "Identifies the WorkItem.", 1739 },
1892 "type": "string" 1740 "description": "The map of transform name prefixes of the jo b to be replaced to the\ncorresponding name prefixes of the new job.",
1893 }, 1741 "type": "object"
1894 "reportIndex": { 1742 },
1895 "description": "The report index. When a WorkItem is leased, the lease will contain an initial report index. When a WorkItem's status is rep orted to the system, the report should be sent with that report index, and the r esponse will contain the index the worker should use for the next report. Report s received with unexpected index values will be rejected by the service. In orde r to preserve idempotency, the worker should not alter the contents of a report, even if the worker must submit the same report multiple times before getting ba ck a response. The worker should not submit a subsequent report until the respon se for the previous report had been received from the service.", 1743 "createTime": {
1896 "format": "int64", 1744 "description": "The timestamp when the job was initially cre ated. Immutable and set by the\nCloud Dataflow service.",
1897 "type": "string" 1745 "format": "google-datetime",
1898 }, 1746 "type": "string"
1899 "requestedLeaseDuration": { 1747 },
1900 "description": "Amount of time the worker requests for its l ease.", 1748 "environment": {
1901 "type": "string" 1749 "$ref": "Environment",
1902 }, 1750 "description": "The environment for the job."
1903 "completed": { 1751 },
1904 "description": "True if the WorkItem was completed (successf ully or unsuccessfully).", 1752 "labels": {
1905 "type": "boolean" 1753 "additionalProperties": {
1906 }, 1754 "type": "string"
1907 "errors": { 1755 },
1908 "description": "Specifies errors which occurred during proce ssing. If errors are provided, and completed = true, then the WorkItem is consid ered to have failed.", 1756 "description": "User-defined labels for this job.\n\nThe lab els map can contain no more than 64 entries. Entries of the labels\nmap are UTF 8 strings that comply with the following restrictions:\n\n* Keys must conform to regexp: \\p{Ll}\\p{Lo}{0,62}\n* Values must conform to regexp: [\\p{Ll}\\p{Lo }\\p{N}_-]{0,63}\n* Both keys and values are additionally constrained to be <= 1 28 bytes in\nsize.",
1909 "items": { 1757 "type": "object"
1910 "$ref": "Status" 1758 },
1911 }, 1759 "type": {
1912 "type": "array" 1760 "description": "The type of Cloud Dataflow job.",
1913 }, 1761 "enum": [
1914 "counterUpdates": { 1762 "JOB_TYPE_UNKNOWN",
1915 "description": "Worker output counters for this WorkItem.", 1763 "JOB_TYPE_BATCH",
1916 "items": { 1764 "JOB_TYPE_STREAMING"
1917 "$ref": "CounterUpdate" 1765 ],
1918 }, 1766 "enumDescriptions": [
1919 "type": "array" 1767 "The type of the job is unspecified, or unknown.",
1920 }, 1768 "A batch job with a well-defined end point: data is read , data is\nprocessed, data is written, and the job is done.",
1921 "metricUpdates": { 1769 "A continuously streaming job with no end: data is read, \nprocessed, and written continuously."
1922 "description": "DEPRECATED in favor of counter_updates.", 1770 ],
1923 "items": { 1771 "type": "string"
1924 "$ref": "MetricUpdate" 1772 },
1925 }, 1773 "projectId": {
1926 "type": "array" 1774 "description": "The ID of the Cloud Platform project that th e job belongs to.",
1927 }, 1775 "type": "string"
1928 "reportedProgress": { 1776 },
1929 "$ref": "ApproximateReportedProgress", 1777 "replaceJobId": {
1930 "description": "The worker's progress through this WorkItem. " 1778 "description": "If this job is an update of an existing job, this field is the job ID\nof the job it replaced.\n\nWhen sending a `CreateJobR equest`, you can update a job by specifying it\nhere. The job named here is stop ped, and its intermediate state is\ntransferred to this job.",
1931 }, 1779 "type": "string"
1932 "stopPosition": { 1780 }
1781 },
1782 "type": "object"
1783 },
1784 "DynamicSourceSplit": {
1785 "description": "When a task splits using WorkItemStatus.dynamic_sour ce_split, this\nmessage describes the two parts of the split relative to the\nde scription of the current task's input.",
1786 "id": "DynamicSourceSplit",
1787 "properties": {
1788 "residual": {
1789 "$ref": "DerivedSource",
1790 "description": "Residual part (returned to the pool of work) .\nSpecified relative to the previously-current source."
1791 },
1792 "primary": {
1793 "$ref": "DerivedSource",
1794 "description": "Primary part (continued to be processed by w orker).\nSpecified relative to the previously-current source.\nBecomes current."
1795 }
1796 },
1797 "type": "object"
1798 },
1799 "DerivedSource": {
1800 "description": "Specification of one of the bundles produced as a re sult of splitting\na Source (e.g. when executing a SourceSplitRequest, or when\n splitting an active task using WorkItemStatus.dynamic_source_split),\nrelative t o the source being split.",
1801 "id": "DerivedSource",
1802 "properties": {
1803 "source": {
1804 "$ref": "Source",
1805 "description": "Specification of the source."
1806 },
1807 "derivationMode": {
1808 "description": "What source to base the produced source on ( if any).",
1809 "enum": [
1810 "SOURCE_DERIVATION_MODE_UNKNOWN",
1811 "SOURCE_DERIVATION_MODE_INDEPENDENT",
1812 "SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT",
1813 "SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT"
1814 ],
1815 "enumDescriptions": [
1816 "The source derivation is unknown, or unspecified.",
1817 "Produce a completely independent Source with no base.",
1818 "Produce a Source based on the Source being split.",
1819 "Produce a Source based on the base of the Source being split."
1820 ],
1821 "type": "string"
1822 }
1823 },
1824 "type": "object"
1825 },
1826 "SourceOperationResponse": {
1827 "description": "The result of a SourceOperationRequest, specified in \nReportWorkItemStatusRequest.source_operation when the work item\nis completed. ",
1828 "id": "SourceOperationResponse",
1829 "properties": {
1830 "split": {
1831 "$ref": "SourceSplitResponse",
1832 "description": "A response to a request to split a source."
1833 },
1834 "getMetadata": {
1835 "$ref": "SourceGetMetadataResponse",
1836 "description": "A response to a request to get metadata abou t a source."
1837 }
1838 },
1839 "type": "object"
1840 },
1841 "SendDebugCaptureResponse": {
1842 "description": "Response to a send capture request.\nnothing",
1843 "id": "SendDebugCaptureResponse",
1844 "properties": {},
1845 "type": "object"
1846 },
1847 "SideInputInfo": {
1848 "description": "Information about a side input of a DoFn or an input of a SeqDoFn.",
1849 "id": "SideInputInfo",
1850 "properties": {
1851 "sources": {
1852 "description": "The source(s) to read element(s) from to get the value of this side input.\nIf more than one source, then the elements are t aken from the\nsources, in the specified order if order matters.\nAt least one s ource is required.",
1853 "items": {
1854 "$ref": "Source"
1855 },
1856 "type": "array"
1857 },
1858 "kind": {
1859 "additionalProperties": {
1860 "description": "Properties of the object.",
1861 "type": "any"
1862 },
1863 "description": "How to interpret the source element(s) as a side input value.",
1864 "type": "object"
1865 },
1866 "tag": {
1867 "description": "The id of the tag the user code will access this side input by;\nthis should correspond to the tag of some MultiOutputInfo." ,
1868 "type": "string"
1869 }
1870 },
1871 "type": "object"
1872 },
1873 "ConcatPosition": {
1874 "description": "A position that encapsulates an inner position and a n index for the inner\nposition. A ConcatPosition can be used by a reader of a s ource that\nencapsulates a set of other sources.",
1875 "id": "ConcatPosition",
1876 "properties": {
1877 "position": {
1933 "$ref": "Position", 1878 "$ref": "Position",
1934 "description": "A worker may split an active map task in two parts, \"primary\" and \"residual\", continuing to process the primary part and returning the residual part into the pool of available work. This event is call ed a \"dynamic split\" and is critical to the dynamic work rebalancing feature. The two obtained sub-tasks are called \"parts\" of the split. The parts, if conc atenated, must represent the same input as would be read by the current task if the split did not happen. The exact way in which the original task is decomposed into the two parts is specified either as a position demarcating them (stop_pos ition), or explicitly as two DerivedSources, if this task consumes a user-define d source type (dynamic_source_split). The \"current\" task is adjusted as a resu lt of the split: after a task with range [A, B) sends a stop_position update at C, its range is considered to be [A, C), e.g.: * Progress should be interpreted relative to the new range, e.g. \"75% completed\" means \"75% of [A, C) complete d\" * The worker should interpret proposed_stop_position relative to the new ran ge, e.g. \"split at 68%\" should be interpreted as \"split at 68% of [A, C)\". * If the worker chooses to split again using stop_position, only stop_positions i n [A, C) will be accepted. * Etc. dynamic_source_split has similar semantics: e. g., if a task with source S splits using dynamic_source_split into {P, R} (where P and R must be together equivalent to S), then subsequent progress and propose d_stop_position should be interpreted relative to P, and in a potential subseque nt dynamic_source_split into {P', R'}, P' and R' must be together equivalent to P, etc." 1879 "description": "Position within the inner source."
1935 }, 1880 },
1936 "dynamicSourceSplit": { 1881 "index": {
1937 "$ref": "DynamicSourceSplit", 1882 "description": "Index of the inner source.",
1938 "description": "See documentation of stop_position."
1939 },
1940 "sourceOperationResponse": {
1941 "$ref": "SourceOperationResponse",
1942 "description": "If the work item represented a SourceOperati onRequest, and the work is completed, contains the result of the operation."
1943 },
1944 "sourceFork": {
1945 "$ref": "SourceFork",
1946 "description": "DEPRECATED in favor of dynamic_source_split. "
1947 },
1948 "progress": {
1949 "$ref": "ApproximateProgress",
1950 "description": "DEPRECATED in favor of reported_progress."
1951 }
1952 },
1953 "type": "object"
1954 },
1955 "Status": {
1956 "description": "The `Status` type defines a logical error model that is suitable for different programming environments, including REST APIs and RPC APIs. It is used by [gRPC](https://github.com/grpc). The error model is designe d to be: - Simple to use and understand for most users - Flexible enough to meet unexpected needs # Overview The `Status` message contains three pieces of data: error code, error message, and error details. The error code should be an enum value of google.rpc.Code, but it may accept additional error codes if needed. Th e error message should be a developer-facing English message that helps develope rs *understand* and *resolve* the error. If a localized user-facing error messag e is needed, put the localized message in the error details or localize it in th e client. The optional error details may contain arbitrary information about the error. There is a predefined set of error detail types in the package `google.r pc` which can be used for common error conditions. # Language mapping The `Statu s` message is the logical representation of the error model, but it is not neces sarily the actual wire format. When the `Status` message is exposed in different client libraries and different wire protocols, it can be mapped differently. Fo r example, it will likely be mapped to some exceptions in Java, but more likely mapped to some error codes in C. # Other uses The error model and the `Status` m essage can be used in a variety of environments, either with or without APIs, to provide a consistent developer experience across different environments. Exampl e uses of this error model include: - Partial errors. If a service needs to retu rn partial errors to the client, it may embed the `Status` in the normal respons e to indicate the partial errors. - Workflow errors. A typical workflow has mult iple steps. Each step may have a `Status` message for error reporting purpose. - Batch operations. If a client uses batch request and batch response, the `Statu s` message should be used directly inside batch response, one for each error sub -response. - Asynchronous operations. If an API call embeds asynchronous operati on results in its response, the status of those operations should be represented directly using the `Status` message. - Logging. If some API errors are stored i n logs, the message `Status` could be used directly after any stripping needed f or security/privacy reasons.",
1957 "id": "Status",
1958 "properties": {
1959 "code": {
1960 "description": "The status code, which should be an enum val ue of google.rpc.Code.",
1961 "format": "int32", 1883 "format": "int32",
1962 "type": "integer" 1884 "type": "integer"
1963 }, 1885 }
1964 "message": { 1886 },
1965 "description": "A developer-facing error message, which shou ld be in English. Any user-facing error message should be localized and sent in the google.rpc.Status.details field, or localized by the client.", 1887 "type": "object"
1966 "type": "string" 1888 },
1967 }, 1889 "WriteInstruction": {
1968 "details": { 1890 "description": "An instruction that writes records.\nTakes one input , produces no outputs.",
1969 "description": "A list of messages that carry the error deta ils. There will be a common set of message types for APIs to use.", 1891 "id": "WriteInstruction",
1970 "items": { 1892 "properties": {
1971 "additionalProperties": { 1893 "input": {
1972 "description": "Properties of the object. Contains f ield @type with type URL.", 1894 "$ref": "InstructionInput",
1973 "type": "any" 1895 "description": "The input."
1974 }, 1896 },
1975 "type": "object" 1897 "sink": {
1976 }, 1898 "$ref": "Sink",
1977 "type": "array" 1899 "description": "The sink to write to."
1978 } 1900 }
1979 }, 1901 },
1980 "type": "object" 1902 "type": "object"
1981 }, 1903 },
1982 "CounterUpdate": { 1904 "CounterStructuredNameAndMetadata": {
1983 "description": "An update to a Counter sent from a worker.", 1905 "description": "A single message which encapsulates structured name and metadata for a given\ncounter.",
1984 "id": "CounterUpdate", 1906 "id": "CounterStructuredNameAndMetadata",
1985 "properties": {
1986 "nameAndKind": {
1987 "$ref": "NameAndKind",
1988 "description": "Counter name and aggregation type."
1989 },
1990 "shortId": {
1991 "description": "The service-generated short identifier for t his counter. The short_id -> (name, metadata) mapping is constant for the lifeti me of a job.",
1992 "format": "int64",
1993 "type": "string"
1994 },
1995 "structuredNameAndMetadata": {
1996 "$ref": "CounterStructuredNameAndMetadata",
1997 "description": "Counter structured name and metadata."
1998 },
1999 "cumulative": {
2000 "description": "True if this counter is reported as the tota l cumulative aggregate value accumulated since the worker started working on thi s WorkItem. By default this is false, indicating that this counter is reported a s a delta.",
2001 "type": "boolean"
2002 },
2003 "integer": {
2004 "$ref": "SplitInt64",
2005 "description": "Integer value for Sum, Max, Min."
2006 },
2007 "floatingPoint": {
2008 "description": "Floating point value for Sum, Max, Min.",
2009 "format": "double",
2010 "type": "number"
2011 },
2012 "boolean": {
2013 "description": "Boolean value for And, Or.",
2014 "type": "boolean"
2015 },
2016 "integerMean": {
2017 "$ref": "IntegerMean",
2018 "description": "Integer mean aggregation value for Mean."
2019 },
2020 "floatingPointMean": {
2021 "$ref": "FloatingPointMean",
2022 "description": "Floating point mean aggregation value for Me an."
2023 },
2024 "integerList": {
2025 "$ref": "IntegerList",
2026 "description": "List of integers, for Set."
2027 },
2028 "floatingPointList": {
2029 "$ref": "FloatingPointList",
2030 "description": "List of floating point numbers, for Set."
2031 },
2032 "stringList": {
2033 "$ref": "StringList",
2034 "description": "List of strings, for Set."
2035 },
2036 "distribution": {
2037 "$ref": "DistributionUpdate",
2038 "description": "Distribution data"
2039 },
2040 "internal": {
2041 "description": "Value for internally-defined counters used b y the Dataflow service.",
2042 "type": "any"
2043 }
2044 },
2045 "type": "object"
2046 },
2047 "NameAndKind": {
2048 "description": "Basic metadata about a counter.",
2049 "id": "NameAndKind",
2050 "properties": { 1907 "properties": {
2051 "name": { 1908 "name": {
2052 "description": "Name of the counter.",
2053 "type": "string"
2054 },
2055 "kind": {
2056 "description": "Counter aggregation kind.",
2057 "enum": [
2058 "INVALID",
2059 "SUM",
2060 "MAX",
2061 "MIN",
2062 "MEAN",
2063 "OR",
2064 "AND",
2065 "SET",
2066 "DISTRIBUTION"
2067 ],
2068 "type": "string"
2069 }
2070 },
2071 "type": "object"
2072 },
2073 "CounterStructuredNameAndMetadata": {
2074 "description": "A single message which encapsulates structured name and metadata for a given counter.",
2075 "id": "CounterStructuredNameAndMetadata",
2076 "properties": {
2077 "name": {
2078 "$ref": "CounterStructuredName", 1909 "$ref": "CounterStructuredName",
2079 "description": "Structured name of the counter." 1910 "description": "Structured name of the counter."
2080 }, 1911 },
2081 "metadata": { 1912 "metadata": {
2082 "$ref": "CounterMetadata", 1913 "$ref": "CounterMetadata",
2083 "description": "Metadata associated with a counter" 1914 "description": "Metadata associated with a counter"
2084 } 1915 }
2085 }, 1916 },
2086 "type": "object" 1917 "type": "object"
2087 }, 1918 },
2088 "CounterStructuredName": { 1919 "AutoscalingSettings": {
2089 "description": "Identifies a counter within a per-job namespace. Cou nters whose structured names are the same get merged into a single value for the job.", 1920 "description": "Settings for WorkerPool autoscaling.",
2090 "id": "CounterStructuredName", 1921 "id": "AutoscalingSettings",
2091 "properties": { 1922 "properties": {
2092 "name": { 1923 "algorithm": {
2093 "description": "Counter name. Not necessarily globally-uniqu e, but unique within the context of the other fields. Required.", 1924 "description": "The algorithm to use for autoscaling.",
2094 "type": "string"
2095 },
2096 "standardOrigin": {
2097 "description": "One of the standard Origins defined above.",
2098 "enum": [ 1925 "enum": [
2099 "DATAFLOW", 1926 "AUTOSCALING_ALGORITHM_UNKNOWN",
2100 "USER" 1927 "AUTOSCALING_ALGORITHM_NONE",
2101 ], 1928 "AUTOSCALING_ALGORITHM_BASIC"
2102 "type": "string" 1929 ],
2103 }, 1930 "enumDescriptions": [
2104 "otherOrigin": { 1931 "The algorithm is unknown, or unspecified.",
2105 "description": "A string containing the origin of the counte r.", 1932 "Disable autoscaling.",
2106 "type": "string" 1933 "Increase worker count over time to reduce job execution time."
2107 }, 1934 ],
2108 "originalStepName": { 1935 "type": "string"
2109 "description": "System generated name of the original step i n the user's graph, before optimization.", 1936 },
2110 "type": "string" 1937 "maxNumWorkers": {
2111 }, 1938 "description": "The maximum number of workers to cap scaling at.",
2112 "componentStepName": {
2113 "description": "Name of the optimized step being executed by the workers.",
2114 "type": "string"
2115 },
2116 "executionStepName": {
2117 "description": "Name of the stage. An execution step contain s multiple component steps.",
2118 "type": "string"
2119 },
2120 "workerId": {
2121 "description": "ID of a particular worker.",
2122 "type": "string"
2123 },
2124 "portion": {
2125 "description": "Portion of this counter, either key or value .",
2126 "enum": [
2127 "ALL",
2128 "KEY",
2129 "VALUE"
2130 ],
2131 "type": "string"
2132 }
2133 },
2134 "type": "object"
2135 },
2136 "CounterMetadata": {
2137 "description": "CounterMetadata includes all static non-name non-val ue counter attributes.",
2138 "id": "CounterMetadata",
2139 "properties": {
2140 "kind": {
2141 "description": "Counter aggregation kind.",
2142 "enum": [
2143 "INVALID",
2144 "SUM",
2145 "MAX",
2146 "MIN",
2147 "MEAN",
2148 "OR",
2149 "AND",
2150 "SET",
2151 "DISTRIBUTION"
2152 ],
2153 "type": "string"
2154 },
2155 "description": {
2156 "description": "Human-readable description of the counter se mantics.",
2157 "type": "string"
2158 },
2159 "standardUnits": {
2160 "description": "System defined Units, see above enum.",
2161 "enum": [
2162 "BYTES",
2163 "BYTES_PER_SEC",
2164 "MILLISECONDS",
2165 "MICROSECONDS",
2166 "NANOSECONDS",
2167 "TIMESTAMP_MSEC",
2168 "TIMESTAMP_USEC",
2169 "TIMESTAMP_NSEC"
2170 ],
2171 "type": "string"
2172 },
2173 "otherUnits": {
2174 "description": "A string referring to the unit type.",
2175 "type": "string"
2176 }
2177 },
2178 "type": "object"
2179 },
2180 "SplitInt64": {
2181 "description": "A representation of an int64, n, that is immune to p recision loss when encoded in JSON.",
2182 "id": "SplitInt64",
2183 "properties": {
2184 "lowBits": {
2185 "description": "The low order bits: n & 0xffffffff.",
2186 "format": "uint32",
2187 "type": "integer"
2188 },
2189 "highBits": {
2190 "description": "The high order bits, including the sign: n > > 32.",
2191 "format": "int32", 1939 "format": "int32",
2192 "type": "integer" 1940 "type": "integer"
2193 } 1941 }
2194 }, 1942 },
2195 "type": "object" 1943 "type": "object"
2196 }, 1944 },
2197 "IntegerMean": { 1945 "StreamingComputationRanges": {
2198 "description": "A representation of an integer mean metric contribut ion.", 1946 "description": "Describes full or partial data disk assignment infor mation of the computation\nranges.",
2199 "id": "IntegerMean", 1947 "id": "StreamingComputationRanges",
2200 "properties": { 1948 "properties": {
2201 "sum": { 1949 "rangeAssignments": {
2202 "$ref": "SplitInt64", 1950 "description": "Data disk assignments for ranges from this c omputation.",
2203 "description": "The sum of all values being aggregated." 1951 "items": {
2204 }, 1952 "$ref": "KeyRangeDataDiskAssignment"
2205 "count": { 1953 },
2206 "$ref": "SplitInt64", 1954 "type": "array"
2207 "description": "The number of values being aggregated." 1955 },
2208 } 1956 "computationId": {
2209 }, 1957 "description": "The ID of the computation.",
2210 "type": "object" 1958 "type": "string"
2211 }, 1959 }
2212 "FloatingPointMean": { 1960 },
2213 "description": "A representation of a floating point mean metric con tribution.", 1961 "type": "object"
2214 "id": "FloatingPointMean", 1962 },
2215 "properties": { 1963 "SendWorkerMessagesRequest": {
2216 "sum": { 1964 "description": "A request for sending worker messages to the service .",
2217 "description": "The sum of all values being aggregated.", 1965 "id": "SendWorkerMessagesRequest",
2218 "format": "double", 1966 "properties": {
2219 "type": "number" 1967 "workerMessages": {
2220 }, 1968 "description": "The WorkerMessages to send.",
2221 "count": { 1969 "items": {
2222 "$ref": "SplitInt64", 1970 "$ref": "WorkerMessage"
2223 "description": "The number of values being aggregated."
2224 }
2225 },
2226 "type": "object"
2227 },
2228 "IntegerList": {
2229 "description": "A metric value representing a list of integers.",
2230 "id": "IntegerList",
2231 "properties": {
2232 "elements": {
2233 "description": "Elements of the list.",
2234 "items": {
2235 "$ref": "SplitInt64"
2236 },
2237 "type": "array"
2238 }
2239 },
2240 "type": "object"
2241 },
2242 "FloatingPointList": {
2243 "description": "A metric value representing a list of floating point numbers.",
2244 "id": "FloatingPointList",
2245 "properties": {
2246 "elements": {
2247 "description": "Elements of the list.",
2248 "items": {
2249 "format": "double",
2250 "type": "number"
2251 },
2252 "type": "array"
2253 }
2254 },
2255 "type": "object"
2256 },
2257 "StringList": {
2258 "description": "A metric value representing a list of strings.",
2259 "id": "StringList",
2260 "properties": {
2261 "elements": {
2262 "description": "Elements of the list.",
2263 "items": {
2264 "type": "string"
2265 },
2266 "type": "array"
2267 }
2268 },
2269 "type": "object"
2270 },
2271 "DistributionUpdate": {
2272 "description": "A metric value representing a distribution.",
2273 "id": "DistributionUpdate",
2274 "properties": {
2275 "min": {
2276 "$ref": "SplitInt64",
2277 "description": "The minimum value present in the distributio n."
2278 },
2279 "max": {
2280 "$ref": "SplitInt64",
2281 "description": "The maximum value present in the distributio n."
2282 },
2283 "count": {
2284 "$ref": "SplitInt64",
2285 "description": "The count of the number of elements present in the distribution."
2286 },
2287 "sum": {
2288 "$ref": "SplitInt64",
2289 "description": "Use an int64 since we'd prefer the added pre cision. If overflow is a common problem we can detect it and use an additional i nt64 or a double."
2290 },
2291 "sumOfSquares": {
2292 "description": "Use a double since the sum of squares is lik ely to overflow int64.",
2293 "format": "double",
2294 "type": "number"
2295 }
2296 },
2297 "type": "object"
2298 },
2299 "ApproximateReportedProgress": {
2300 "description": "A progress measurement of a WorkItem by a worker.",
2301 "id": "ApproximateReportedProgress",
2302 "properties": {
2303 "position": {
2304 "$ref": "Position",
2305 "description": "A Position within the work to represent a pr ogress."
2306 },
2307 "fractionConsumed": {
2308 "description": "Completion as fraction of the input consumed , from 0.0 (beginning, nothing consumed), to 1.0 (end of the input, entire input consumed).",
2309 "format": "double",
2310 "type": "number"
2311 },
2312 "remainingParallelism": {
2313 "$ref": "ReportedParallelism",
2314 "description": "Total amount of parallelism in the input of this task that remains, (i.e. can be delegated to this task and any new tasks vi a dynamic splitting). Always at least 1 for non-finished work items and 0 for fi nished. \"Amount of parallelism\" refers to how many non-empty parts of the inpu t can be read in parallel. This does not necessarily equal number of records. An input that can be read in parallel down to the individual records is called \"p erfectly splittable\". An example of non-perfectly parallelizable input is a blo ck-compressed file format where a block of records has to be read as a whole, bu t different blocks can be read in parallel. Examples: * If we are processing rec ord #30 (starting at 1) out of 50 in a perfectly splittable 50-record input, thi s value should be 21 (20 remaining + 1 current). * If we are reading through blo ck 3 in a block-compressed file consisting of 5 blocks, this value should be 3 ( since blocks 4 and 5 can be processed in parallel by new tasks via dynamic split ting and the current task remains processing block 3). * If we are reading throu gh the last block in a block-compressed file, or reading or processing the last record in a perfectly splittable input, this value should be 1, because apart fr om the current task, no additional remainder can be split off."
2315 },
2316 "consumedParallelism": {
2317 "$ref": "ReportedParallelism",
2318 "description": "Total amount of parallelism in the portion o f input of this task that has already been consumed and is no longer active. In the first two examples above (see remaining_parallelism), the value should be 29 or 2 respectively. The sum of remaining_parallelism and consumed_parallelism sh ould equal the total amount of parallelism in this work item. If specified, must be finite."
2319 }
2320 },
2321 "type": "object"
2322 },
2323 "Position": {
2324 "description": "Position defines a position within a collection of d ata. The value can be either the end position, a key (used with ordered collecti ons), a byte offset, or a record index.",
2325 "id": "Position",
2326 "properties": {
2327 "end": {
2328 "description": "Position is past all other positions. Also u seful for the end position of an unbounded range.",
2329 "type": "boolean"
2330 },
2331 "key": {
2332 "description": "Position is a string key, ordered lexicograp hically.",
2333 "type": "string"
2334 },
2335 "byteOffset": {
2336 "description": "Position is a byte offset.",
2337 "format": "int64",
2338 "type": "string"
2339 },
2340 "recordIndex": {
2341 "description": "Position is a record index.",
2342 "format": "int64",
2343 "type": "string"
2344 },
2345 "shufflePosition": {
2346 "description": "CloudPosition is a base64 encoded BatchShuff lePosition (with FIXED sharding).",
2347 "type": "string"
2348 },
2349 "concatPosition": {
2350 "$ref": "ConcatPosition",
2351 "description": "CloudPosition is a concat position."
2352 }
2353 },
2354 "type": "object"
2355 },
2356 "ConcatPosition": {
2357 "description": "A position that encapsulates an inner position and a n index for the inner position. A ConcatPosition can be used by a reader of a so urce that encapsulates a set of other sources.",
2358 "id": "ConcatPosition",
2359 "properties": {
2360 "index": {
2361 "description": "Index of the inner source.",
2362 "format": "int32",
2363 "type": "integer"
2364 },
2365 "position": {
2366 "$ref": "Position",
2367 "description": "Position within the inner source."
2368 }
2369 },
2370 "type": "object"
2371 },
2372 "ReportedParallelism": {
2373 "description": "Represents the level of parallelism in a WorkItem's input, reported by the worker.",
2374 "id": "ReportedParallelism",
2375 "properties": {
2376 "isInfinite": {
2377 "description": "Specifies whether the parallelism is infinit e. If true, \"value\" is ignored. Infinite parallelism means the service will as sume that the work item can always be split into more non-empty work items by dy namic splitting. This is a work-around for lack of support for infinity by the c urrent JSON-based Java RPC stack.",
2378 "type": "boolean"
2379 },
2380 "value": {
2381 "description": "Specifies the level of parallelism in case i t is finite.",
2382 "format": "double",
2383 "type": "number"
2384 }
2385 },
2386 "type": "object"
2387 },
2388 "DynamicSourceSplit": {
2389 "description": "When a task splits using WorkItemStatus.dynamic_sour ce_split, this message describes the two parts of the split relative to the desc ription of the current task's input.",
2390 "id": "DynamicSourceSplit",
2391 "properties": {
2392 "primary": {
2393 "$ref": "DerivedSource",
2394 "description": "Primary part (continued to be processed by w orker). Specified relative to the previously-current source. Becomes current."
2395 },
2396 "residual": {
2397 "$ref": "DerivedSource",
2398 "description": "Residual part (returned to the pool of work) . Specified relative to the previously-current source."
2399 }
2400 },
2401 "type": "object"
2402 },
2403 "DerivedSource": {
2404 "description": "Specification of one of the bundles produced as a re sult of splitting a Source (e.g. when executing a SourceSplitRequest, or when sp litting an active task using WorkItemStatus.dynamic_source_split), relative to t he source being split.",
2405 "id": "DerivedSource",
2406 "properties": {
2407 "source": {
2408 "$ref": "Source",
2409 "description": "Specification of the source."
2410 },
2411 "derivationMode": {
2412 "description": "What source to base the produced source on ( if any).",
2413 "enum": [
2414 "SOURCE_DERIVATION_MODE_UNKNOWN",
2415 "SOURCE_DERIVATION_MODE_INDEPENDENT",
2416 "SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT",
2417 "SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT"
2418 ],
2419 "type": "string"
2420 }
2421 },
2422 "type": "object"
2423 },
2424 "Source": {
2425 "description": "A source that records can be read and decoded from." ,
2426 "id": "Source",
2427 "properties": {
2428 "spec": {
2429 "additionalProperties": {
2430 "description": "Properties of the object.",
2431 "type": "any"
2432 },
2433 "description": "The source to read from, plus its parameters .",
2434 "type": "object"
2435 },
2436 "codec": {
2437 "additionalProperties": {
2438 "description": "Properties of the object.",
2439 "type": "any"
2440 },
2441 "description": "The codec to use to decode data read from th e source.",
2442 "type": "object"
2443 },
2444 "baseSpecs": {
2445 "description": "While splitting, sources may specify the pro duced bundles as differences against another source, in order to save backend-si de memory and allow bigger jobs. For details, see SourceSplitRequest. To support this use case, the full set of parameters of the source is logically obtained b y taking the latest explicitly specified value of each parameter in the order: b ase_specs (later items win), spec (overrides anything in base_specs).",
2446 "items": {
2447 "additionalProperties": {
2448 "description": "Properties of the object.",
2449 "type": "any"
2450 },
2451 "type": "object"
2452 },
2453 "type": "array"
2454 },
2455 "metadata": {
2456 "$ref": "SourceMetadata",
2457 "description": "Optionally, metadata for this source can be supplied right away, avoiding a SourceGetMetadataOperation roundtrip (see Source OperationRequest). This field is meaningful only in the Source objects populated by the user (e.g. when filling in a DerivedSource). Source objects supplied by the framework to the user don't have this field populated."
2458 },
2459 "doesNotNeedSplitting": {
2460 "description": "Setting this value to true hints to the fram ework that the source doesn't need splitting, and using SourceSplitRequest on it would yield SOURCE_SPLIT_OUTCOME_USE_CURRENT. E.g. a file splitter may set this to true when splitting a single file into a set of byte ranges of appropriate s ize, and set this to false when splitting a filepattern into individual files. H owever, for efficiency, a file splitter may decide to produce file subranges dir ectly from the filepattern to avoid a splitting round-trip. See SourceSplitReque st for an overview of the splitting process. This field is meaningful only in th e Source objects populated by the user (e.g. when filling in a DerivedSource). S ource objects supplied by the framework to the user don't have this field popula ted.",
2461 "type": "boolean"
2462 }
2463 },
2464 "type": "object"
2465 },
2466 "SourceMetadata": {
2467 "description": "Metadata about a Source useful for automatically opt imizing and tuning the pipeline, etc.",
2468 "id": "SourceMetadata",
2469 "properties": {
2470 "producesSortedKeys": {
2471 "description": "Whether this source is known to produce key/ value pairs with the (encoded) keys in lexicographically sorted order.",
2472 "type": "boolean"
2473 },
2474 "infinite": {
2475 "description": "Specifies that the size of this source is kn own to be infinite (this is a streaming source).",
2476 "type": "boolean"
2477 },
2478 "estimatedSizeBytes": {
2479 "description": "An estimate of the total size (in bytes) of the data that would be read from this source. This estimate is in terms of exter nal storage size, before any decompression or other processing done by the reade r.",
2480 "format": "int64",
2481 "type": "string"
2482 }
2483 },
2484 "type": "object"
2485 },
2486 "SourceOperationResponse": {
2487 "description": "The result of a SourceOperationRequest, specified in ReportWorkItemStatusRequest.source_operation when the work item is completed.",
2488 "id": "SourceOperationResponse",
2489 "properties": {
2490 "split": {
2491 "$ref": "SourceSplitResponse",
2492 "description": "A response to a request to split a source."
2493 },
2494 "getMetadata": {
2495 "$ref": "SourceGetMetadataResponse",
2496 "description": "A response to a request to get metadata abou t a source."
2497 }
2498 },
2499 "type": "object"
2500 },
2501 "SourceSplitResponse": {
2502 "description": "The response to a SourceSplitRequest.",
2503 "id": "SourceSplitResponse",
2504 "properties": {
2505 "outcome": {
2506 "description": "Indicates whether splitting happened and pro duced a list of bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current source should be processed \"as is\" without splitting. \"bundles\" is ignored in this case. If this is SPLITTING_HAPPENED, then \"bundles\" contains a list of bundle s into which the source was split.",
2507 "enum": [
2508 "SOURCE_SPLIT_OUTCOME_UNKNOWN",
2509 "SOURCE_SPLIT_OUTCOME_USE_CURRENT",
2510 "SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED"
2511 ],
2512 "type": "string"
2513 },
2514 "bundles": {
2515 "description": "If outcome is SPLITTING_HAPPENED, then this is a list of bundles into which the source was split. Otherwise this field is ig nored. This list can be empty, which means the source represents an empty input. ",
2516 "items": {
2517 "$ref": "DerivedSource"
2518 },
2519 "type": "array"
2520 },
2521 "shards": {
2522 "description": "DEPRECATED in favor of bundles.",
2523 "items": {
2524 "$ref": "SourceSplitShard"
2525 }, 1971 },
2526 "type": "array" 1972 "type": "array"
2527 } 1973 }
2528 }, 1974 },
2529 "type": "object" 1975 "type": "object"
2530 }, 1976 },
2531 "SourceSplitShard": { 1977 "SourceSplitShard": {
2532 "description": "DEPRECATED in favor of DerivedSource.", 1978 "description": "DEPRECATED in favor of DerivedSource.",
2533 "id": "SourceSplitShard", 1979 "id": "SourceSplitShard",
2534 "properties": { 1980 "properties": {
2535 "source": {
2536 "$ref": "Source",
2537 "description": "DEPRECATED"
2538 },
2539 "derivationMode": { 1981 "derivationMode": {
2540 "description": "DEPRECATED", 1982 "description": "DEPRECATED",
2541 "enum": [ 1983 "enum": [
2542 "SOURCE_DERIVATION_MODE_UNKNOWN", 1984 "SOURCE_DERIVATION_MODE_UNKNOWN",
2543 "SOURCE_DERIVATION_MODE_INDEPENDENT", 1985 "SOURCE_DERIVATION_MODE_INDEPENDENT",
2544 "SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT", 1986 "SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT",
2545 "SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT" 1987 "SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT"
2546 ], 1988 ],
2547 "type": "string" 1989 "enumDescriptions": [
1990 "The source derivation is unknown, or unspecified.",
1991 "Produce a completely independent Source with no base.",
1992 "Produce a Source based on the Source being split.",
1993 "Produce a Source based on the base of the Source being split."
1994 ],
1995 "type": "string"
1996 },
1997 "source": {
1998 "$ref": "Source",
1999 "description": "DEPRECATED"
2000 }
2001 },
2002 "type": "object"
2003 },
2004 "CPUTime": {
2005 "description": "Modeled after information exposed by /proc/stat.",
2006 "id": "CPUTime",
2007 "properties": {
2008 "totalMs": {
2009 "description": "Total active CPU time across all cores (ie., non-idle) in milliseconds\nsince start-up.",
2010 "format": "uint64",
2011 "type": "string"
2012 },
2013 "rate": {
2014 "description": "Average CPU utilization rate (% non-idle cpu / second) since previous\nsample.",
2015 "format": "double",
2016 "type": "number"
2017 },
2018 "timestamp": {
2019 "description": "Timestamp of the measurement.",
2020 "format": "google-datetime",
2021 "type": "string"
2022 }
2023 },
2024 "type": "object"
2025 },
2026 "Environment": {
2027 "description": "Describes the environment in which a Dataflow Job ru ns.",
2028 "id": "Environment",
2029 "properties": {
2030 "clusterManagerApiService": {
2031 "description": "The type of cluster manager API to use. If unknown or\nunspecified, the service will attempt to choose a reasonable\ndefaul t. This should be in the form of the API service name,\ne.g. \"compute.googleap is.com\".",
2032 "type": "string"
2033 },
2034 "tempStoragePrefix": {
2035 "description": "The prefix of the resources the system shoul d use for temporary\nstorage. The system will append the suffix \"/temp-{JOBNAM E} to\nthis resource prefix, where {JOBNAME} is the value of the\njob_name field . The resulting bucket and object prefix is used\nas the prefix of the resource s used to store temporary data\nneeded during the job execution. NOTE: This wil l override the\nvalue in taskrunner_settings.\nThe supported resource type is:\n \nGoogle Cloud Storage:\n\n storage.googleapis.com/{bucket}/{object}\n bucket. storage.googleapis.com/{object}",
2036 "type": "string"
2037 },
2038 "workerPools": {
2039 "description": "The worker pools. At least one \"harness\" w orker pool must be\nspecified in order for the job to have workers.",
2040 "items": {
2041 "$ref": "WorkerPool"
2042 },
2043 "type": "array"
2044 },
2045 "dataset": {
2046 "description": "The dataset for the current project where va rious workflow\nrelated tables are stored.\n\nThe supported resource type is:\n\ nGoogle BigQuery:\n bigquery.googleapis.com/{dataset}",
2047 "type": "string"
2048 },
2049 "experiments": {
2050 "description": "The list of experiments to enable.",
2051 "items": {
2052 "type": "string"
2053 },
2054 "type": "array"
2055 },
2056 "internalExperiments": {
2057 "additionalProperties": {
2058 "description": "Properties of the object. Contains field @type with type URL.",
2059 "type": "any"
2060 },
2061 "description": "Experimental settings.",
2062 "type": "object"
2063 },
2064 "version": {
2065 "additionalProperties": {
2066 "description": "Properties of the object.",
2067 "type": "any"
2068 },
2069 "description": "A structure describing which components and their versions of the service\nare required in order to run the job.",
2070 "type": "object"
2071 },
2072 "serviceAccountEmail": {
2073 "description": "Identity to run virtual machines as. Default s to the default account.",
2074 "type": "string"
2075 },
2076 "userAgent": {
2077 "additionalProperties": {
2078 "description": "Properties of the object.",
2079 "type": "any"
2080 },
2081 "description": "A description of the process that generated the request.",
2082 "type": "object"
2083 },
2084 "sdkPipelineOptions": {
2085 "additionalProperties": {
2086 "description": "Properties of the object.",
2087 "type": "any"
2088 },
2089 "description": "The Cloud Dataflow SDK pipeline options spec ified by the user. These\noptions are passed through the service and are used to recreate the\nSDK pipeline options on the worker in a language agnostic and pla tform\nindependent way.",
2090 "type": "object"
2091 }
2092 },
2093 "type": "object"
2094 },
2095 "StreamingComputationTask": {
2096 "description": "A task which describes what action should be perform ed for the specified\nstreaming computation ranges.",
2097 "id": "StreamingComputationTask",
2098 "properties": {
2099 "dataDisks": {
2100 "description": "Describes the set of data disks this task sh ould apply to.",
2101 "items": {
2102 "$ref": "MountedDataDisk"
2103 },
2104 "type": "array"
2105 },
2106 "taskType": {
2107 "description": "A type of streaming computation task.",
2108 "enum": [
2109 "STREAMING_COMPUTATION_TASK_UNKNOWN",
2110 "STREAMING_COMPUTATION_TASK_STOP",
2111 "STREAMING_COMPUTATION_TASK_START"
2112 ],
2113 "enumDescriptions": [
2114 "The streaming computation task is unknown, or unspecifi ed.",
2115 "Stop processing specified streaming computation range(s ).",
2116 "Start processing specified streaming computation range( s)."
2117 ],
2118 "type": "string"
2119 },
2120 "computationRanges": {
2121 "description": "Contains ranges of a streaming computation t his task should apply to.",
2122 "items": {
2123 "$ref": "StreamingComputationRanges"
2124 },
2125 "type": "array"
2126 }
2127 },
2128 "type": "object"
2129 },
2130 "SendDebugCaptureRequest": {
2131 "description": "Request to send encoded debug information.",
2132 "id": "SendDebugCaptureRequest",
2133 "properties": {
2134 "componentId": {
2135 "description": "The internal component id for which debug in formation is sent.",
2136 "type": "string"
2137 },
2138 "workerId": {
2139 "description": "The worker id, i.e., VM hostname.",
2140 "type": "string"
2141 },
2142 "data": {
2143 "description": "The encoded debug information.",
2144 "type": "string"
2145 }
2146 },
2147 "type": "object"
2148 },
2149 "GetDebugConfigResponse": {
2150 "description": "Response to a get debug configuration request.",
2151 "id": "GetDebugConfigResponse",
2152 "properties": {
2153 "config": {
2154 "description": "The encoded debug configuration for the requ ested component.",
2155 "type": "string"
2156 }
2157 },
2158 "type": "object"
2159 },
2160 "StreamingSetupTask": {
2161 "description": "A task which initializes part of a streaming Dataflo w job.",
2162 "id": "StreamingSetupTask",
2163 "properties": {
2164 "workerHarnessPort": {
2165 "description": "The TCP port used by the worker to communica te with the Dataflow\nworker harness.",
2166 "format": "int32",
2167 "type": "integer"
2168 },
2169 "drain": {
2170 "description": "The user has requested drain.",
2171 "type": "boolean"
2172 },
2173 "receiveWorkPort": {
2174 "description": "The TCP port on which the worker should list en for messages from\nother streaming computation workers.",
2175 "format": "int32",
2176 "type": "integer"
2177 },
2178 "streamingComputationTopology": {
2179 "$ref": "TopologyConfig",
2180 "description": "The global topology of the streaming Dataflo w job."
2181 }
2182 },
2183 "type": "object"
2184 },
2185 "PubsubLocation": {
2186 "description": "Identifies a pubsub location to use for transferring data into or\nout of a streaming Dataflow job.",
2187 "id": "PubsubLocation",
2188 "properties": {
2189 "withAttributes": {
2190 "description": "If true, then the client has requested to ge t pubsub attributes.",
2191 "type": "boolean"
2192 },
2193 "idLabel": {
2194 "description": "If set, contains a pubsub label from which t o extract record ids.\nIf left empty, record deduplication will be strictly best effort.",
2195 "type": "string"
2196 },
2197 "topic": {
2198 "description": "A pubsub topic, in the form of\n\"pubsub.goo gleapis.com/topics/<project-id>/<topic-name>\"",
2199 "type": "string"
2200 },
2201 "timestampLabel": {
2202 "description": "If set, contains a pubsub label from which t o extract record timestamps.\nIf left empty, record timestamps will be generated upon arrival.",
2203 "type": "string"
2204 },
2205 "subscription": {
2206 "description": "A pubsub subscription, in the form of\n\"pub sub.googleapis.com/subscriptions/<project-id>/<subscription-name>\"",
2207 "type": "string"
2208 },
2209 "dropLateData": {
2210 "description": "Indicates whether the pipeline allows late-a rriving data.",
2211 "type": "boolean"
2212 },
2213 "trackingSubscription": {
2214 "description": "If set, specifies the pubsub subscription th at will be used for tracking\ncustom time timestamps for watermark estimation.",
2215 "type": "string"
2216 }
2217 },
2218 "type": "object"
2219 },
2220 "WorkerHealthReport": {
2221 "description": "WorkerHealthReport contains information about the he alth of a worker.\n\nThe VM should be identified by the labels attached to the W orkerMessage that\nthis health ping belongs to.",
2222 "id": "WorkerHealthReport",
2223 "properties": {
2224 "pods": {
2225 "description": "The pods running on the worker. See:\nhttp:/ /kubernetes.io/v1.1/docs/api-reference/v1/definitions.html#_v1_pod\n\nThis field is used by the worker to send the status of the indvidual\ncontainers running o n each worker.",
2226 "items": {
2227 "additionalProperties": {
2228 "description": "Properties of the object.",
2229 "type": "any"
2230 },
2231 "type": "object"
2232 },
2233 "type": "array"
2234 },
2235 "vmStartupTime": {
2236 "description": "The time the VM was booted.",
2237 "format": "google-datetime",
2238 "type": "string"
2239 },
2240 "vmIsHealthy": {
2241 "description": "Whether the VM is healthy.",
2242 "type": "boolean"
2243 },
2244 "reportInterval": {
2245 "description": "The interval at which the worker is sending health reports.\nThe default value of 0 should be interpreted as the field is no t being\nexplicitly set by the worker.",
2246 "format": "google-duration",
2247 "type": "string"
2248 }
2249 },
2250 "type": "object"
2251 },
2252 "JobMessage": {
2253 "description": "A particular message pertaining to a Dataflow job.",
2254 "id": "JobMessage",
2255 "properties": {
2256 "id": {
2257 "description": "Identifies the message. This is automatical ly generated by the\nservice; the caller should treat it as an opaque string.",
2258 "type": "string"
2259 },
2260 "messageText": {
2261 "description": "The text of the message.",
2262 "type": "string"
2263 },
2264 "messageImportance": {
2265 "description": "Importance level of the message.",
2266 "enum": [
2267 "JOB_MESSAGE_IMPORTANCE_UNKNOWN",
2268 "JOB_MESSAGE_DEBUG",
2269 "JOB_MESSAGE_DETAILED",
2270 "JOB_MESSAGE_BASIC",
2271 "JOB_MESSAGE_WARNING",
2272 "JOB_MESSAGE_ERROR"
2273 ],
2274 "enumDescriptions": [
2275 "The message importance isn't specified, or is unknown." ,
2276 "The message is at the 'debug' level: typically only use ful for\nsoftware engineers working on the code the job is running.\nTypically, Dataflow pipeline runners do not display log messages\nat this level by default. ",
2277 "The message is at the 'detailed' level: somewhat verbos e, but\npotentially useful to users. Typically, Dataflow pipeline\nrunners do n ot display log messages at this level by default.\nThese messages are displayed by default in the Dataflow\nmonitoring UI.",
2278 "The message is at the 'basic' level: useful for keeping \ntrack of the execution of a Dataflow pipeline. Typically,\nDataflow pipeline runners display log messages at this level by\ndefault, and these messages are d isplayed by default in the\nDataflow monitoring UI.",
2279 "The message is at the 'warning' level: indicating a con dition\npertaining to a job which may require human intervention.\nTypically, Da taflow pipeline runners display log messages at this\nlevel by default, and thes e messages are displayed by default in\nthe Dataflow monitoring UI.",
2280 "The message is at the 'error' level: indicating a condi tion\npreventing a job from succeeding. Typically, Dataflow pipeline\nrunners d isplay log messages at this level by default, and these\nmessages are displayed by default in the Dataflow monitoring UI."
2281 ],
2282 "type": "string"
2283 },
2284 "time": {
2285 "description": "The timestamp of the message.",
2286 "format": "google-datetime",
2287 "type": "string"
2288 }
2289 },
2290 "type": "object"
2291 },
2292 "MultiOutputInfo": {
2293 "description": "Information about an output of a multi-output DoFn." ,
2294 "id": "MultiOutputInfo",
2295 "properties": {
2296 "tag": {
2297 "description": "The id of the tag the user code will emit to this output by; this\nshould correspond to the tag of some SideInputInfo.",
2298 "type": "string"
2299 }
2300 },
2301 "type": "object"
2302 },
2303 "SourceSplitRequest": {
2304 "description": "Represents the operation to split a high-level Sourc e specification\ninto bundles (parts for parallel processing).\n\nAt a high leve l, splitting of a source into bundles happens as follows:\nSourceSplitRequest is applied to the source. If it returns\nSOURCE_SPLIT_OUTCOME_USE_CURRENT, no furt her splitting happens and the source\nis used \"as is\". Otherwise, splitting is applied recursively to each\nproduced DerivedSource.\n\nAs an optimization, for any Source, if its does_not_need_splitting is\ntrue, the framework assumes that splitting this source would return\nSOURCE_SPLIT_OUTCOME_USE_CURRENT, and doesn 't initiate a SourceSplitRequest.\nThis applies both to the initial source being split and to bundles\nproduced from it.",
2305 "id": "SourceSplitRequest",
2306 "properties": {
2307 "source": {
2308 "$ref": "Source",
2309 "description": "Specification of the source to be split."
2310 },
2311 "options": {
2312 "$ref": "SourceSplitOptions",
2313 "description": "Hints for tuning the splitting process."
2548 } 2314 }
2549 }, 2315 },
2550 "type": "object" 2316 "type": "object"
2551 }, 2317 },
2552 "SourceGetMetadataResponse": { 2318 "SourceGetMetadataResponse": {
2553 "description": "The result of a SourceGetMetadataOperation.", 2319 "description": "The result of a SourceGetMetadataOperation.",
2554 "id": "SourceGetMetadataResponse", 2320 "id": "SourceGetMetadataResponse",
2555 "properties": { 2321 "properties": {
2556 "metadata": { 2322 "metadata": {
2557 "$ref": "SourceMetadata", 2323 "$ref": "SourceMetadata",
2558 "description": "The computed metadata." 2324 "description": "The computed metadata."
2559 } 2325 }
2560 }, 2326 },
2561 "type": "object" 2327 "type": "object"
2562 }, 2328 },
2563 "SourceFork": { 2329 "MetricShortId": {
2564 "description": "DEPRECATED in favor of DynamicSourceSplit.", 2330 "description": "The metric short id is returned to the user alongsid e an offset into\nReportWorkItemStatusRequest",
2565 "id": "SourceFork", 2331 "id": "MetricShortId",
2566 "properties": { 2332 "properties": {
2567 "primary": { 2333 "metricIndex": {
2568 "$ref": "SourceSplitShard", 2334 "description": "The index of the corresponding metric in\nth e ReportWorkItemStatusRequest. Required.",
2569 "description": "DEPRECATED" 2335 "format": "int32",
2570 }, 2336 "type": "integer"
2571 "residual": { 2337 },
2572 "$ref": "SourceSplitShard", 2338 "shortId": {
2573 "description": "DEPRECATED" 2339 "description": "The service-generated short identifier for t he metric.",
2574 }, 2340 "format": "int64",
2575 "primarySource": { 2341 "type": "string"
2576 "$ref": "DerivedSource", 2342 }
2577 "description": "DEPRECATED" 2343 },
2578 }, 2344 "type": "object"
2579 "residualSource": { 2345 },
2580 "$ref": "DerivedSource", 2346 "ShellTask": {
2581 "description": "DEPRECATED" 2347 "description": "A task which consists of a shell command for the wor ker to execute.",
2582 } 2348 "id": "ShellTask",
2583 }, 2349 "properties": {
2584 "type": "object" 2350 "exitCode": {
2585 }, 2351 "description": "Exit code for the task.",
2586 "ApproximateProgress": { 2352 "format": "int32",
2587 "description": "Obsolete in favor of ApproximateReportedProgress and ApproximateSplitRequest.", 2353 "type": "integer"
2588 "id": "ApproximateProgress", 2354 },
2589 "properties": { 2355 "command": {
2590 "position": { 2356 "description": "The shell command to run.",
2591 "$ref": "Position", 2357 "type": "string"
2592 "description": "Obsolete." 2358 }
2593 }, 2359 },
2594 "percentComplete": { 2360 "type": "object"
2595 "description": "Obsolete.", 2361 },
2596 "format": "float", 2362 "TaskRunnerSettings": {
2597 "type": "number" 2363 "description": "Taskrunner configuration settings.",
2598 }, 2364 "id": "TaskRunnerSettings",
2599 "remainingTime": { 2365 "properties": {
2600 "description": "Obsolete.", 2366 "baseUrl": {
2601 "type": "string" 2367 "description": "The base URL for the taskrunner to use when accessing Google Cloud APIs.\n\nWhen workers access Google Cloud APIs, they logi cally do so via\nrelative URLs. If this field is specified, it supplies the bas e\nURL to use for resolving these relative URLs. The normative\nalgorithm used is defined by RFC 1808, \"Relative Uniform Resource\nLocators\".\n\nIf not speci fied, the default value is \"http://www.googleapis.com/\"",
2602 } 2368 "type": "string"
2603 }, 2369 },
2604 "type": "object" 2370 "logToSerialconsole": {
2605 }, 2371 "description": "Whether to send taskrunner log info to Googl e Compute Engine VM serial\nconsole.",
2606 "ReportWorkItemStatusResponse": { 2372 "type": "boolean"
2607 "description": "Response from a request to report the status of Work Items.", 2373 },
2608 "id": "ReportWorkItemStatusResponse", 2374 "continueOnException": {
2609 "properties": { 2375 "description": "Whether to continue taskrunner if an excepti on is hit.",
2610 "workItemServiceStates": { 2376 "type": "boolean"
2611 "description": "A set of messages indicating the service-sid e state for each WorkItem whose status was reported, in the same order as the Wo rkItemStatus messages in the ReportWorkItemStatusRequest which resulting in this response.", 2377 },
2612 "items": { 2378 "parallelWorkerSettings": {
2613 "$ref": "WorkItemServiceState" 2379 "$ref": "WorkerSettings",
2614 }, 2380 "description": "The settings to pass to the parallel worker harness."
2615 "type": "array" 2381 },
2616 } 2382 "vmId": {
2617 }, 2383 "description": "The ID string of the VM.",
2618 "type": "object" 2384 "type": "string"
2619 }, 2385 },
2620 "WorkItemServiceState": { 2386 "taskUser": {
2621 "description": "The Dataflow service's idea of the current state of a WorkItem being processed by a worker.", 2387 "description": "The UNIX user ID on the worker VM to use for tasks launched by\ntaskrunner; e.g. \"root\".",
2622 "id": "WorkItemServiceState", 2388 "type": "string"
2623 "properties": { 2389 },
2624 "splitRequest": { 2390 "alsologtostderr": {
2625 "$ref": "ApproximateSplitRequest", 2391 "description": "Whether to also send taskrunner log info to stderr.",
2626 "description": "The progress point in the WorkItem where the Dataflow service suggests that the worker truncate the task." 2392 "type": "boolean"
2627 }, 2393 },
2628 "leaseExpireTime": { 2394 "taskGroup": {
2629 "description": "Time at which the current lease will expire. ", 2395 "description": "The UNIX group ID on the worker VM to use fo r tasks launched by\ntaskrunner; e.g. \"wheel\".",
2630 "type": "string" 2396 "type": "string"
2631 }, 2397 },
2632 "reportStatusInterval": { 2398 "harnessCommand": {
2633 "description": "New recommended reporting interval.", 2399 "description": "The command to launch the worker harness.",
2634 "type": "string" 2400 "type": "string"
2635 }, 2401 },
2636 "harnessData": { 2402 "logDir": {
2403 "description": "The directory on the VM to store logs.",
2404 "type": "string"
2405 },
2406 "dataflowApiVersion": {
2407 "description": "The API version of endpoint, e.g. \"v1b3\"",
2408 "type": "string"
2409 },
2410 "oauthScopes": {
2411 "description": "The OAuth2 scopes to be requested by the tas krunner in order to\naccess the Cloud Dataflow API.",
2412 "items": {
2413 "type": "string"
2414 },
2415 "type": "array"
2416 },
2417 "streamingWorkerMainClass": {
2418 "description": "The streaming worker main class name.",
2419 "type": "string"
2420 },
2421 "logUploadLocation": {
2422 "description": "Indicates where to put logs. If this is not specified, the logs\nwill not be uploaded.\n\nThe supported resource type is:\n \nGoogle Cloud Storage:\n storage.googleapis.com/{bucket}/{object}\n bucket.st orage.googleapis.com/{object}",
2423 "type": "string"
2424 },
2425 "workflowFileName": {
2426 "description": "The file to store the workflow in.",
2427 "type": "string"
2428 },
2429 "commandlinesFileName": {
2430 "description": "The file to store preprocessing commands in. ",
2431 "type": "string"
2432 },
2433 "languageHint": {
2434 "description": "The suggested backend language.",
2435 "type": "string"
2436 },
2437 "tempStoragePrefix": {
2438 "description": "The prefix of the resources the taskrunner s hould use for\ntemporary storage.\n\nThe supported resource type is:\n\nGoogle C loud Storage:\n storage.googleapis.com/{bucket}/{object}\n bucket.storage.goog leapis.com/{object}",
2439 "type": "string"
2440 },
2441 "baseTaskDir": {
2442 "description": "The location on the worker for task-specific subdirectories.",
2443 "type": "string"
2444 }
2445 },
2446 "type": "object"
2447 },
2448 "Position": {
2449 "description": "Position defines a position within a collection of d ata. The value\ncan be either the end position, a key (used with ordered\ncolle ctions), a byte offset, or a record index.",
2450 "id": "Position",
2451 "properties": {
2452 "shufflePosition": {
2453 "description": "CloudPosition is a base64 encoded BatchShuff lePosition (with FIXED\nsharding).",
2454 "type": "string"
2455 },
2456 "concatPosition": {
2457 "$ref": "ConcatPosition",
2458 "description": "CloudPosition is a concat position."
2459 },
2460 "byteOffset": {
2461 "description": "Position is a byte offset.",
2462 "format": "int64",
2463 "type": "string"
2464 },
2465 "end": {
2466 "description": "Position is past all other positions. Also u seful for the end\nposition of an unbounded range.",
2467 "type": "boolean"
2468 },
2469 "key": {
2470 "description": "Position is a string key, ordered lexicograp hically.",
2471 "type": "string"
2472 },
2473 "recordIndex": {
2474 "description": "Position is a record index.",
2475 "format": "int64",
2476 "type": "string"
2477 }
2478 },
2479 "type": "object"
2480 },
2481 "SplitInt64": {
2482 "description": "A representation of an int64, n, that is immune to p recision loss when\nencoded in JSON.",
2483 "id": "SplitInt64",
2484 "properties": {
2485 "lowBits": {
2486 "description": "The low order bits: n & 0xffffffff.",
2487 "format": "uint32",
2488 "type": "integer"
2489 },
2490 "highBits": {
2491 "description": "The high order bits, including the sign: n > > 32.",
2492 "format": "int32",
2493 "type": "integer"
2494 }
2495 },
2496 "type": "object"
2497 },
2498 "Source": {
2499 "description": "A source that records can be read and decoded from." ,
2500 "id": "Source",
2501 "properties": {
2502 "codec": {
2637 "additionalProperties": { 2503 "additionalProperties": {
2638 "description": "Properties of the object.", 2504 "description": "Properties of the object.",
2639 "type": "any" 2505 "type": "any"
2640 }, 2506 },
2641 "description": "Other data returned by the service, specific to the particular worker harness.", 2507 "description": "The codec to use to decode data read from th e source.",
2642 "type": "object" 2508 "type": "object"
2643 }, 2509 },
2644 "nextReportIndex": { 2510 "doesNotNeedSplitting": {
2645 "description": "The index value to use for the next report s ent by the worker. Note: If the report call fails for whatever reason, the worke r should reuse this index for subsequent report attempts.", 2511 "description": "Setting this value to true hints to the fram ework that the source\ndoesn't need splitting, and using SourceSplitRequest on i t would\nyield SOURCE_SPLIT_OUTCOME_USE_CURRENT.\n\nE.g. a file splitter may set this to true when splitting a single file\ninto a set of byte ranges of appropr iate size, and set this\nto false when splitting a filepattern into individual f iles.\nHowever, for efficiency, a file splitter may decide to produce\nfile subr anges directly from the filepattern to avoid a splitting\nround-trip.\n\nSee Sou rceSplitRequest for an overview of the splitting process.\n\nThis field is meani ngful only in the Source objects populated\nby the user (e.g. when filling in a DerivedSource).\nSource objects supplied by the framework to the user don't have \nthis field populated.",
2512 "type": "boolean"
2513 },
2514 "spec": {
2515 "additionalProperties": {
2516 "description": "Properties of the object.",
2517 "type": "any"
2518 },
2519 "description": "The source to read from, plus its parameters .",
2520 "type": "object"
2521 },
2522 "metadata": {
2523 "$ref": "SourceMetadata",
2524 "description": "Optionally, metadata for this source can be supplied right away,\navoiding a SourceGetMetadataOperation roundtrip\n(see Sour ceOperationRequest).\n\nThis field is meaningful only in the Source objects popu lated\nby the user (e.g. when filling in a DerivedSource).\nSource objects suppl ied by the framework to the user don't have\nthis field populated."
2525 },
2526 "baseSpecs": {
2527 "description": "While splitting, sources may specify the pro duced bundles\nas differences against another source, in order to save backend-s ide\nmemory and allow bigger jobs. For details, see SourceSplitRequest.\nTo supp ort this use case, the full set of parameters of the source\nis logically obtain ed by taking the latest explicitly specified value\nof each parameter in the ord er:\nbase_specs (later items win), spec (overrides anything in base_specs).",
2528 "items": {
2529 "additionalProperties": {
2530 "description": "Properties of the object.",
2531 "type": "any"
2532 },
2533 "type": "object"
2534 },
2535 "type": "array"
2536 }
2537 },
2538 "type": "object"
2539 },
2540 "WorkerPool": {
2541 "description": "Describes one particular pool of Cloud Dataflow work ers to be\ninstantiated by the Cloud Dataflow service in order to perform the\nc omputations required by a job. Note that a workflow job may use\nmultiple pools , in order to match the various computational\nrequirements of the various stage s of the job.",
2542 "id": "WorkerPool",
2543 "properties": {
2544 "workerHarnessContainerImage": {
2545 "description": "Required. Docker container image that execut es the Cloud Dataflow worker\nharness, residing in Google Container Registry.",
2546 "type": "string"
2547 },
2548 "diskType": {
2549 "description": "Type of root disk for VMs. If empty or unsp ecified, the service will\nattempt to choose a reasonable default.",
2550 "type": "string"
2551 },
2552 "machineType": {
2553 "description": "Machine type (e.g. \"n1-standard-1\"). If e mpty or unspecified, the\nservice will attempt to choose a reasonable default.",
2554 "type": "string"
2555 },
2556 "kind": {
2557 "description": "The kind of the worker pool; currently only `harness` and `shuffle`\nare supported.",
2558 "type": "string"
2559 },
2560 "dataDisks": {
2561 "description": "Data disks that are used by a VM in this wor kflow.",
2562 "items": {
2563 "$ref": "Disk"
2564 },
2565 "type": "array"
2566 },
2567 "subnetwork": {
2568 "description": "Subnetwork to which VMs will be assigned, if desired. Expected to be of\nthe form \"regions/REGION/subnetworks/SUBNETWORK\" .",
2569 "type": "string"
2570 },
2571 "ipConfiguration": {
2572 "description": "Configuration for VM IPs.",
2573 "enum": [
2574 "WORKER_IP_UNSPECIFIED",
2575 "WORKER_IP_PUBLIC",
2576 "WORKER_IP_PRIVATE"
2577 ],
2578 "enumDescriptions": [
2579 "The configuration is unknown, or unspecified.",
2580 "Workers should have public IP addresses.",
2581 "Workers should have private IP addresses."
2582 ],
2583 "type": "string"
2584 },
2585 "autoscalingSettings": {
2586 "$ref": "AutoscalingSettings",
2587 "description": "Settings for autoscaling of this WorkerPool. "
2588 },
2589 "taskrunnerSettings": {
2590 "$ref": "TaskRunnerSettings",
2591 "description": "Settings passed through to Google Compute En gine workers when\nusing the standard Dataflow task runner. Users should ignore \nthis field."
2592 },
2593 "metadata": {
2594 "additionalProperties": {
2595 "type": "string"
2596 },
2597 "description": "Metadata to set on the Google Compute Engine VMs.",
2598 "type": "object"
2599 },
2600 "network": {
2601 "description": "Network to which VMs will be assigned. If e mpty or unspecified,\nthe service will use the network \"default\".",
2602 "type": "string"
2603 },
2604 "defaultPackageSet": {
2605 "description": "The default package set to install. This al lows the service to\nselect a default set of packages which are useful to worker \nharnesses written in a particular language.",
2606 "enum": [
2607 "DEFAULT_PACKAGE_SET_UNKNOWN",
2608 "DEFAULT_PACKAGE_SET_NONE",
2609 "DEFAULT_PACKAGE_SET_JAVA",
2610 "DEFAULT_PACKAGE_SET_PYTHON"
2611 ],
2612 "enumDescriptions": [
2613 "The default set of packages to stage is unknown, or uns pecified.",
2614 "Indicates that no packages should be staged at the work er unless\nexplicitly specified by the job.",
2615 "Stage packages typically useful to workers written in J ava.",
2616 "Stage pacakges typically useful to workers written in P ython."
2617 ],
2618 "type": "string"
2619 },
2620 "numThreadsPerWorker": {
2621 "description": "The number of threads per worker harness. If empty or unspecified, the\nservice will choose a number of threads (according t o the number of cores\non the selected machine type for batch, or 1 by conventio n for streaming).",
2622 "format": "int32",
2623 "type": "integer"
2624 },
2625 "numWorkers": {
2626 "description": "Number of Google Compute Engine workers in t his pool needed to\nexecute the job. If zero or unspecified, the service will\n attempt to choose a reasonable default.",
2627 "format": "int32",
2628 "type": "integer"
2629 },
2630 "zone": {
2631 "description": "Zone to run the worker pools in. If empty o r unspecified, the service\nwill attempt to choose a reasonable default.",
2632 "type": "string"
2633 },
2634 "diskSourceImage": {
2635 "description": "Fully qualified source image for disks.",
2636 "type": "string"
2637 },
2638 "packages": {
2639 "description": "Packages to be installed on workers.",
2640 "items": {
2641 "$ref": "Package"
2642 },
2643 "type": "array"
2644 },
2645 "teardownPolicy": {
2646 "description": "Sets the policy for determining when to turn down worker pool.\nAllowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and\n`TEARDOWN_NEVER`.\n`TEARDOWN_ALWAYS` means workers are always torn down re gardless of whether\nthe job succeeds. `TEARDOWN_ON_SUCCESS` means workers are t orn down\nif the job succeeds. `TEARDOWN_NEVER` means the workers are never torn \ndown.\n\nIf the workers are not torn down by the service, they will\ncontinue to run and use Google Compute Engine VM resources in the\nuser's project until t hey are explicitly terminated by the user.\nBecause of this, Google recommends u sing the `TEARDOWN_ALWAYS`\npolicy except for small, manually supervised test jo bs.\n\nIf unknown or unspecified, the service will attempt to choose a reasonabl e\ndefault.",
2647 "enum": [
2648 "TEARDOWN_POLICY_UNKNOWN",
2649 "TEARDOWN_ALWAYS",
2650 "TEARDOWN_ON_SUCCESS",
2651 "TEARDOWN_NEVER"
2652 ],
2653 "enumDescriptions": [
2654 "The teardown policy isn't specified, or is unknown.",
2655 "Always teardown the resource.",
2656 "Teardown the resource on success. This is useful for de bugging\nfailures.",
2657 "Never teardown the resource. This is useful for debuggi ng and\ndevelopment."
2658 ],
2659 "type": "string"
2660 },
2661 "onHostMaintenance": {
2662 "description": "The action to take on host maintenance, as d efined by the Google\nCompute Engine API.",
2663 "type": "string"
2664 },
2665 "poolArgs": {
2666 "additionalProperties": {
2667 "description": "Properties of the object. Contains field @type with type URL.",
2668 "type": "any"
2669 },
2670 "description": "Extra arguments for this worker pool.",
2671 "type": "object"
2672 },
2673 "diskSizeGb": {
2674 "description": "Size of root disk for VMs, in GB. If zero o r unspecified, the service will\nattempt to choose a reasonable default.",
2675 "format": "int32",
2676 "type": "integer"
2677 }
2678 },
2679 "type": "object"
2680 },
2681 "SourceOperationRequest": {
2682 "description": "A work item that represents the different operations that can be\nperformed on a user-defined Source specification.",
2683 "id": "SourceOperationRequest",
2684 "properties": {
2685 "split": {
2686 "$ref": "SourceSplitRequest",
2687 "description": "Information about a request to split a sourc e."
2688 },
2689 "getMetadata": {
2690 "$ref": "SourceGetMetadataRequest",
2691 "description": "Information about a request to get metadata about a source."
2692 }
2693 },
2694 "type": "object"
2695 },
2696 "WorkItem": {
2697 "description": "WorkItem represents basic information about a WorkIt em to be executed\nin the cloud.",
2698 "id": "WorkItem",
2699 "properties": {
2700 "initialReportIndex": {
2701 "description": "The initial index to use when reporting the status of the WorkItem.",
2646 "format": "int64", 2702 "format": "int64",
2647 "type": "string" 2703 "type": "string"
2648 }, 2704 },
2649 "metricShortId": { 2705 "streamingComputationTask": {
2650 "description": "The short ids that workers should use in sub sequent metric updates. Workers should strive to use short ids whenever possible , but it is ok to request the short_id again if a worker lost track of it (e.g. if the worker is recovering from a crash). NOTE: it is possible that the respons e may have short ids for a subset of the metrics.", 2706 "$ref": "StreamingComputationTask",
2651 "items": { 2707 "description": "Additional information for StreamingComputat ionTask WorkItems."
2652 "$ref": "MetricShortId" 2708 },
2653 }, 2709 "shellTask": {
2654 "type": "array" 2710 "$ref": "ShellTask",
2655 }, 2711 "description": "Additional information for ShellTask WorkIte ms."
2656 "suggestedStopPosition": { 2712 },
2657 "$ref": "Position", 2713 "jobId": {
2658 "description": "Obsolete, always empty." 2714 "description": "Identifies the workflow job this WorkItem be longs to.",
2659 }, 2715 "type": "string"
2660 "suggestedStopPoint": { 2716 },
2661 "$ref": "ApproximateProgress",
2662 "description": "DEPRECATED in favor of split_request."
2663 }
2664 },
2665 "type": "object"
2666 },
2667 "ApproximateSplitRequest": {
2668 "description": "A suggestion by the service to the worker to dynamic ally split the WorkItem.",
2669 "id": "ApproximateSplitRequest",
2670 "properties": {
2671 "position": {
2672 "$ref": "Position",
2673 "description": "A Position at which to split the work item."
2674 },
2675 "fractionConsumed": {
2676 "description": "A fraction at which to split the work item, from 0.0 (beginning of the input) to 1.0 (end of the input).",
2677 "format": "double",
2678 "type": "number"
2679 }
2680 },
2681 "type": "object"
2682 },
2683 "MetricShortId": {
2684 "description": "The metric short id is returned to the user alongsid e an offset into ReportWorkItemStatusRequest",
2685 "id": "MetricShortId",
2686 "properties": {
2687 "metricIndex": {
2688 "description": "The index of the corresponding metric in the ReportWorkItemStatusRequest. Required.",
2689 "format": "int32",
2690 "type": "integer"
2691 },
2692 "shortId": {
2693 "description": "The service-generated short identifier for t he metric.",
2694 "format": "int64",
2695 "type": "string"
2696 }
2697 },
2698 "type": "object"
2699 },
2700 "LeaseWorkItemRequest": {
2701 "description": "Request to lease WorkItems.",
2702 "id": "LeaseWorkItemRequest",
2703 "properties": {
2704 "workItemTypes": {
2705 "description": "Filter for WorkItem type.",
2706 "items": {
2707 "type": "string"
2708 },
2709 "type": "array"
2710 },
2711 "workerCapabilities": {
2712 "description": "Worker capabilities. WorkItems might be limi ted to workers with specific capabilities.",
2713 "items": {
2714 "type": "string"
2715 },
2716 "type": "array"
2717 },
2718 "requestedLeaseDuration": {
2719 "description": "The initial lease period.",
2720 "type": "string"
2721 },
2722 "currentWorkerTime": {
2723 "description": "The current timestamp at the worker.",
2724 "type": "string"
2725 },
2726 "workerId": {
2727 "description": "Identifies the worker leasing work -- typica lly the ID of the virtual machine running the worker.",
2728 "type": "string"
2729 },
2730 "location": {
2731 "description": "The location which contains the WorkItem's j ob.",
2732 "type": "string"
2733 }
2734 },
2735 "type": "object"
2736 },
2737 "LeaseWorkItemResponse": {
2738 "description": "Response to a request to lease WorkItems.",
2739 "id": "LeaseWorkItemResponse",
2740 "properties": {
2741 "workItems": {
2742 "description": "A list of the leased WorkItems.",
2743 "items": {
2744 "$ref": "WorkItem"
2745 },
2746 "type": "array"
2747 }
2748 },
2749 "type": "object"
2750 },
2751 "WorkItem": {
2752 "description": "WorkItem represents basic information about a WorkIt em to be executed in the cloud.",
2753 "id": "WorkItem",
2754 "properties": {
2755 "id": { 2717 "id": {
2756 "description": "Identifies this WorkItem.", 2718 "description": "Identifies this WorkItem.",
2757 "format": "int64", 2719 "format": "int64",
2758 "type": "string" 2720 "type": "string"
2759 }, 2721 },
2760 "projectId": { 2722 "configuration": {
2761 "description": "Identifies the cloud project this WorkItem b elongs to.", 2723 "description": "Work item-specific configuration as an opaqu e blob.",
2762 "type": "string" 2724 "type": "string"
2763 },
2764 "jobId": {
2765 "description": "Identifies the workflow job this WorkItem be longs to.",
2766 "type": "string"
2767 },
2768 "packages": {
2769 "description": "Any required packages that need to be fetche d in order to execute this WorkItem.",
2770 "items": {
2771 "$ref": "Package"
2772 },
2773 "type": "array"
2774 }, 2725 },
2775 "mapTask": { 2726 "mapTask": {
2776 "$ref": "MapTask", 2727 "$ref": "MapTask",
2777 "description": "Additional information for MapTask WorkItems ." 2728 "description": "Additional information for MapTask WorkItems ."
2778 }, 2729 },
2779 "seqMapTask": { 2730 "seqMapTask": {
2780 "$ref": "SeqMapTask", 2731 "$ref": "SeqMapTask",
2781 "description": "Additional information for SeqMapTask WorkIt ems." 2732 "description": "Additional information for SeqMapTask WorkIt ems."
2782 }, 2733 },
2783 "shellTask": { 2734 "packages": {
2784 "$ref": "ShellTask", 2735 "description": "Any required packages that need to be fetche d in order to execute\nthis WorkItem.",
2785 "description": "Additional information for ShellTask WorkIte ms." 2736 "items": {
2737 "$ref": "Package"
2738 },
2739 "type": "array"
2740 },
2741 "projectId": {
2742 "description": "Identifies the cloud project this WorkItem b elongs to.",
2743 "type": "string"
2744 },
2745 "reportStatusInterval": {
2746 "description": "Recommended reporting interval.",
2747 "format": "google-duration",
2748 "type": "string"
2749 },
2750 "sourceOperationTask": {
2751 "$ref": "SourceOperationRequest",
2752 "description": "Additional information for source operation WorkItems."
2786 }, 2753 },
2787 "streamingSetupTask": { 2754 "streamingSetupTask": {
2788 "$ref": "StreamingSetupTask", 2755 "$ref": "StreamingSetupTask",
2789 "description": "Additional information for StreamingSetupTas k WorkItems." 2756 "description": "Additional information for StreamingSetupTas k WorkItems."
2790 }, 2757 },
2791 "sourceOperationTask": {
2792 "$ref": "SourceOperationRequest",
2793 "description": "Additional information for source operation WorkItems."
2794 },
2795 "streamingComputationTask": {
2796 "$ref": "StreamingComputationTask",
2797 "description": "Additional information for StreamingComputat ionTask WorkItems."
2798 },
2799 "streamingConfigTask": { 2758 "streamingConfigTask": {
2800 "$ref": "StreamingConfigTask", 2759 "$ref": "StreamingConfigTask",
2801 "description": "Additional information for StreamingConfigTa sk WorkItems." 2760 "description": "Additional information for StreamingConfigTa sk WorkItems."
2802 }, 2761 },
2803 "reportStatusInterval": {
2804 "description": "Recommended reporting interval.",
2805 "type": "string"
2806 },
2807 "leaseExpireTime": { 2762 "leaseExpireTime": {
2808 "description": "Time when the lease on this Work will expire .", 2763 "description": "Time when the lease on this Work will expire .",
2809 "type": "string" 2764 "format": "google-datetime",
2810 }, 2765 "type": "string"
2811 "configuration": { 2766 }
2812 "description": "Work item-specific configuration as an opaqu e blob.", 2767 },
2813 "type": "string" 2768 "type": "object"
2814 }, 2769 },
2815 "initialReportIndex": { 2770 "ResourceUtilizationReport": {
2816 "description": "The initial index to use when reporting the status of the WorkItem.", 2771 "description": "Worker metrics exported from workers. This contains resource utilization\nmetrics accumulated from a variety of sources. For more in formation, see\ngo/df-resource-signals.",
2772 "id": "ResourceUtilizationReport",
2773 "properties": {
2774 "cpuTime": {
2775 "description": "CPU utilization samples.",
2776 "items": {
2777 "$ref": "CPUTime"
2778 },
2779 "type": "array"
2780 }
2781 },
2782 "type": "object"
2783 },
2784 "ReportedParallelism": {
2785 "description": "Represents the level of parallelism in a WorkItem's input,\nreported by the worker.",
2786 "id": "ReportedParallelism",
2787 "properties": {
2788 "isInfinite": {
2789 "description": "Specifies whether the parallelism is infinit e. If true, \"value\" is\nignored.\nInfinite parallelism means the service will assume that the work item\ncan always be split into more non-empty work items by dynamic splitting.\nThis is a work-around for lack of support for infinity by t he current\nJSON-based Java RPC stack.",
2790 "type": "boolean"
2791 },
2792 "value": {
2793 "description": "Specifies the level of parallelism in case i t is finite.",
2794 "format": "double",
2795 "type": "number"
2796 }
2797 },
2798 "type": "object"
2799 },
2800 "TopologyConfig": {
2801 "description": "Global topology of the streaming Dataflow job, inclu ding all\ncomputations and their sharded locations.",
2802 "id": "TopologyConfig",
2803 "properties": {
2804 "forwardingKeyBits": {
2805 "description": "The size (in bits) of keys that will be assi gned to source messages.",
2806 "format": "int32",
2807 "type": "integer"
2808 },
2809 "userStageToComputationNameMap": {
2810 "additionalProperties": {
2811 "type": "string"
2812 },
2813 "description": "Maps user stage names to stable computation names.",
2814 "type": "object"
2815 },
2816 "computations": {
2817 "description": "The computations associated with a streaming Dataflow job.",
2818 "items": {
2819 "$ref": "ComputationTopology"
2820 },
2821 "type": "array"
2822 },
2823 "dataDiskAssignments": {
2824 "description": "The disks assigned to a streaming Dataflow j ob.",
2825 "items": {
2826 "$ref": "DataDiskAssignment"
2827 },
2828 "type": "array"
2829 },
2830 "persistentStateVersion": {
2831 "description": "Version number for persistent state.",
2832 "format": "int32",
2833 "type": "integer"
2834 }
2835 },
2836 "type": "object"
2837 },
2838 "SourceSplitOptions": {
2839 "description": "Hints for splitting a Source into bundles (parts for parallel\nprocessing) using SourceSplitRequest.",
2840 "id": "SourceSplitOptions",
2841 "properties": {
2842 "desiredBundleSizeBytes": {
2843 "description": "The source should be split into a set of bun dles where the estimated size\nof each is approximately this many bytes.",
2817 "format": "int64", 2844 "format": "int64",
2818 "type": "string" 2845 "type": "string"
2819 } 2846 },
2820 }, 2847 "desiredShardSizeBytes": {
2821 "type": "object" 2848 "description": "DEPRECATED in favor of desired_bundle_size_b ytes.",
2822 }, 2849 "format": "int64",
2823 "MapTask": { 2850 "type": "string"
2824 "description": "MapTask consists of an ordered set of instructions, each of which describes one particular low-level operation for the worker to per form in order to accomplish the MapTask's WorkItem. Each instruction must appear in the list before any instructions which depends on its output.",
2825 "id": "MapTask",
2826 "properties": {
2827 "instructions": {
2828 "description": "The instructions in the MapTask.",
2829 "items": {
2830 "$ref": "ParallelInstruction"
2831 },
2832 "type": "array"
2833 },
2834 "systemName": {
2835 "description": "System-defined name of this MapTask. Unique across the workflow.",
2836 "type": "string"
2837 },
2838 "stageName": {
2839 "description": "System-defined name of the stage containing this MapTask. Unique across the workflow.",
2840 "type": "string"
2841 }
2842 },
2843 "type": "object"
2844 },
2845 "ParallelInstruction": {
2846 "description": "Describes a particular operation comprising a MapTas k.",
2847 "id": "ParallelInstruction",
2848 "properties": {
2849 "systemName": {
2850 "description": "System-defined name of this operation. Uniqu e across the workflow.",
2851 "type": "string"
2852 },
2853 "name": {
2854 "description": "User-provided name of this operation.",
2855 "type": "string"
2856 },
2857 "originalName": {
2858 "description": "System-defined name for the operation in the original workflow graph.",
2859 "type": "string"
2860 },
2861 "read": {
2862 "$ref": "ReadInstruction",
2863 "description": "Additional information for Read instructions ."
2864 },
2865 "write": {
2866 "$ref": "WriteInstruction",
2867 "description": "Additional information for Write instruction s."
2868 },
2869 "parDo": {
2870 "$ref": "ParDoInstruction",
2871 "description": "Additional information for ParDo instruction s."
2872 },
2873 "partialGroupByKey": {
2874 "$ref": "PartialGroupByKeyInstruction",
2875 "description": "Additional information for PartialGroupByKey instructions."
2876 },
2877 "flatten": {
2878 "$ref": "FlattenInstruction",
2879 "description": "Additional information for Flatten instructi ons."
2880 },
2881 "outputs": {
2882 "description": "Describes the outputs of the instruction.",
2883 "items": {
2884 "$ref": "InstructionOutput"
2885 },
2886 "type": "array"
2887 } 2851 }
2888 }, 2852 },
2889 "type": "object" 2853 "type": "object"
2890 }, 2854 },
2891 "ReadInstruction": { 2855 "ReadInstruction": {
2892 "description": "An instruction that reads records. Takes no inputs, produces one output.", 2856 "description": "An instruction that reads records.\nTakes no inputs, produces one output.",
2893 "id": "ReadInstruction", 2857 "id": "ReadInstruction",
2894 "properties": { 2858 "properties": {
2895 "source": { 2859 "source": {
2896 "$ref": "Source", 2860 "$ref": "Source",
2897 "description": "The source to read from." 2861 "description": "The source to read from."
2898 } 2862 }
2899 }, 2863 },
2900 "type": "object" 2864 "type": "object"
2901 }, 2865 },
2902 "WriteInstruction": { 2866 "WorkerSettings": {
2903 "description": "An instruction that writes records. Takes one input, produces no outputs.", 2867 "description": "Provides data to pass through to the worker harness. ",
2904 "id": "WriteInstruction", 2868 "id": "WorkerSettings",
2905 "properties": { 2869 "properties": {
2906 "input": { 2870 "servicePath": {
2907 "$ref": "InstructionInput", 2871 "description": "The Cloud Dataflow service path relative to the root URL, for example,\n\"dataflow/v1b3/projects\".",
2908 "description": "The input." 2872 "type": "string"
2909 }, 2873 },
2910 "sink": { 2874 "shuffleServicePath": {
2911 "$ref": "Sink", 2875 "description": "The Shuffle service path relative to the roo t URL, for example,\n\"shuffle/v1beta1\".",
2912 "description": "The sink to write to." 2876 "type": "string"
2913 } 2877 },
2914 }, 2878 "workerId": {
2915 "type": "object" 2879 "description": "The ID of the worker running this pipeline." ,
2916 }, 2880 "type": "string"
2917 "InstructionInput": { 2881 },
2918 "description": "An input of an instruction, as a reference to an out put of a producer instruction.", 2882 "tempStoragePrefix": {
2919 "id": "InstructionInput", 2883 "description": "The prefix of the resources the system shoul d use for temporary\nstorage.\n\nThe supported resource type is:\n\nGoogle Cloud Storage:\n\n storage.googleapis.com/{bucket}/{object}\n bucket.storage.google apis.com/{object}",
2920 "properties": { 2884 "type": "string"
2921 "producerInstructionIndex": { 2885 },
2922 "description": "The index (origin zero) of the parallel inst ruction that produces the output to be consumed by this input. This index is rel ative to the list of instructions in this input's instruction's containing MapTa sk.", 2886 "baseUrl": {
2887 "description": "The base URL for accessing Google Cloud APIs .\n\nWhen workers access Google Cloud APIs, they logically do so via\nrelative U RLs. If this field is specified, it supplies the base\nURL to use for resolving these relative URLs. The normative\nalgorithm used is defined by RFC 1808, \"R elative Uniform Resource\nLocators\".\n\nIf not specified, the default value is \"http://www.googleapis.com/\"",
2888 "type": "string"
2889 },
2890 "reportingEnabled": {
2891 "description": "Whether to send work progress updates to the service.",
2892 "type": "boolean"
2893 }
2894 },
2895 "type": "object"
2896 },
2897 "DataDiskAssignment": {
2898 "description": "Data disk assignment for a given VM instance.",
2899 "id": "DataDiskAssignment",
2900 "properties": {
2901 "vmInstance": {
2902 "description": "VM instance name the data disks mounted to, for example\n\"myproject-1014-104817-4c2-harness-0\".",
2903 "type": "string"
2904 },
2905 "dataDisks": {
2906 "description": "Mounted data disks. The order is important a data disk's 0-based index in\nthis list defines which persistent directory the disk is mounted to, for\nexample the list of { \"myproject-1014-104817-4c2-harne ss-0-disk-0\" },\n{ \"myproject-1014-104817-4c2-harness-0-disk-1\" }.",
2907 "items": {
2908 "type": "string"
2909 },
2910 "type": "array"
2911 }
2912 },
2913 "type": "object"
2914 },
2915 "StreamingStageLocation": {
2916 "description": "Identifies the location of a streaming computation s tage, for\nstage-to-stage communication.",
2917 "id": "StreamingStageLocation",
2918 "properties": {
2919 "streamId": {
2920 "description": "Identifies the particular stream within the streaming Dataflow\njob.",
2921 "type": "string"
2922 }
2923 },
2924 "type": "object"
2925 },
2926 "ApproximateSplitRequest": {
2927 "description": "A suggestion by the service to the worker to dynamic ally split the WorkItem.",
2928 "id": "ApproximateSplitRequest",
2929 "properties": {
2930 "fractionConsumed": {
2931 "description": "A fraction at which to split the work item, from 0.0 (beginning of the\ninput) to 1.0 (end of the input).",
2932 "format": "double",
2933 "type": "number"
2934 },
2935 "position": {
2936 "$ref": "Position",
2937 "description": "A Position at which to split the work item."
2938 }
2939 },
2940 "type": "object"
2941 },
2942 "Status": {
2943 "description": "The `Status` type defines a logical error model that is suitable for different\nprogramming environments, including REST APIs and RP C APIs. It is used by\n[gRPC](https://github.com/grpc). The error model is desig ned to be:\n\n- Simple to use and understand for most users\n- Flexible enough t o meet unexpected needs\n\n# Overview\n\nThe `Status` message contains three pie ces of data: error code, error message,\nand error details. The error code shoul d be an enum value of\ngoogle.rpc.Code, but it may accept additional error codes if needed. The\nerror message should be a developer-facing English message tha t helps\ndevelopers *understand* and *resolve* the error. If a localized user-fa cing\nerror message is needed, put the localized message in the error details or \nlocalize it in the client. The optional error details may contain arbitrary\ni nformation about the error. There is a predefined set of error detail types\nin the package `google.rpc` which can be used for common error conditions.\n\n# Lan guage mapping\n\nThe `Status` message is the logical representation of the error model, but it\nis not necessarily the actual wire format. When the `Status` mes sage is\nexposed in different client libraries and different wire protocols, it can be\nmapped differently. For example, it will likely be mapped to some except ions\nin Java, but more likely mapped to some error codes in C.\n\n# Other uses\ n\nThe error model and the `Status` message can be used in a variety of\nenviron ments, either with or without APIs, to provide a\nconsistent developer experienc e across different environments.\n\nExample uses of this error model include:\n\ n- Partial errors. If a service needs to return partial errors to the client,\n it may embed the `Status` in the normal response to indicate the partial\n errors.\n\n- Workflow errors. A typical workflow has multiple steps. Each step may\n have a `Status` message for error reporting purpose.\n\n- Batch operati ons. If a client uses batch request and batch response, the\n `Status` messag e should be used directly inside batch response, one for\n each error sub-res ponse.\n\n- Asynchronous operations. If an API call embeds asynchronous operatio n\n results in its response, the status of those operations should be\n re presented directly using the `Status` message.\n\n- Logging. If some API errors are stored in logs, the message `Status` could\n be used directly after any s tripping needed for security/privacy reasons.",
2944 "id": "Status",
2945 "properties": {
2946 "details": {
2947 "description": "A list of messages that carry the error deta ils. There will be a\ncommon set of message types for APIs to use.",
2948 "items": {
2949 "additionalProperties": {
2950 "description": "Properties of the object. Contains f ield @type with type URL.",
2951 "type": "any"
2952 },
2953 "type": "object"
2954 },
2955 "type": "array"
2956 },
2957 "code": {
2958 "description": "The status code, which should be an enum val ue of google.rpc.Code.",
2923 "format": "int32", 2959 "format": "int32",
2924 "type": "integer" 2960 "type": "integer"
2925 }, 2961 },
2926 "outputNum": { 2962 "message": {
2927 "description": "The output index (origin zero) within the pr oducer.", 2963 "description": "A developer-facing error message, which shou ld be in English. Any\nuser-facing error message should be localized and sent in the\ngoogle.rpc.Status.details field, or localized by the client.",
2928 "format": "int32", 2964 "type": "string"
2929 "type": "integer" 2965 }
2966 },
2967 "type": "object"
2968 },
2969 "StreamLocation": {
2970 "description": "Describes a stream of data, either as input to be pr ocessed or as\noutput of a streaming Dataflow job.",
2971 "id": "StreamLocation",
2972 "properties": {
2973 "customSourceLocation": {
2974 "$ref": "CustomSourceLocation",
2975 "description": "The stream is a custom source."
2976 },
2977 "streamingStageLocation": {
2978 "$ref": "StreamingStageLocation",
2979 "description": "The stream is part of another computation wi thin the current\nstreaming Dataflow job."
2980 },
2981 "pubsubLocation": {
2982 "$ref": "PubsubLocation",
2983 "description": "The stream is a pubsub stream."
2984 },
2985 "sideInputLocation": {
2986 "$ref": "StreamingSideInputLocation",
2987 "description": "The stream is a streaming side input."
2988 }
2989 },
2990 "type": "object"
2991 },
2992 "SendWorkerMessagesResponse": {
2993 "description": "The response to the worker messages.",
2994 "id": "SendWorkerMessagesResponse",
2995 "properties": {
2996 "workerMessageResponses": {
2997 "description": "The servers response to the worker messages. ",
2998 "items": {
2999 "$ref": "WorkerMessageResponse"
3000 },
3001 "type": "array"
3002 }
3003 },
3004 "type": "object"
3005 },
3006 "LeaseWorkItemResponse": {
3007 "description": "Response to a request to lease WorkItems.",
3008 "id": "LeaseWorkItemResponse",
3009 "properties": {
3010 "workItems": {
3011 "description": "A list of the leased WorkItems.",
3012 "items": {
3013 "$ref": "WorkItem"
3014 },
3015 "type": "array"
3016 }
3017 },
3018 "type": "object"
3019 },
3020 "StreamingComputationConfig": {
3021 "description": "Configuration information for a single streaming com putation.",
3022 "id": "StreamingComputationConfig",
3023 "properties": {
3024 "computationId": {
3025 "description": "Unique identifier for this computation.",
3026 "type": "string"
3027 },
3028 "systemName": {
3029 "description": "System defined name for this computation.",
3030 "type": "string"
3031 },
3032 "stageName": {
3033 "description": "Stage name of this computation.",
3034 "type": "string"
3035 },
3036 "instructions": {
3037 "description": "Instructions that comprise the computation." ,
3038 "items": {
3039 "$ref": "ParallelInstruction"
3040 },
3041 "type": "array"
2930 } 3042 }
2931 }, 3043 },
2932 "type": "object" 3044 "type": "object"
2933 }, 3045 },
2934 "Sink": { 3046 "Sink": {
2935 "description": "A sink that records can be encoded and written to.", 3047 "description": "A sink that records can be encoded and written to.",
2936 "id": "Sink", 3048 "id": "Sink",
2937 "properties": { 3049 "properties": {
3050 "codec": {
3051 "additionalProperties": {
3052 "description": "Properties of the object.",
3053 "type": "any"
3054 },
3055 "description": "The codec to use to encode data written to t he sink.",
3056 "type": "object"
3057 },
2938 "spec": { 3058 "spec": {
2939 "additionalProperties": { 3059 "additionalProperties": {
2940 "description": "Properties of the object.", 3060 "description": "Properties of the object.",
2941 "type": "any" 3061 "type": "any"
2942 }, 3062 },
2943 "description": "The sink to write to, plus its parameters.", 3063 "description": "The sink to write to, plus its parameters.",
2944 "type": "object" 3064 "type": "object"
2945 }, 3065 }
2946 "codec": { 3066 },
2947 "additionalProperties": { 3067 "type": "object"
2948 "description": "Properties of the object.",
2949 "type": "any"
2950 },
2951 "description": "The codec to use to encode data written to t he sink.",
2952 "type": "object"
2953 }
2954 },
2955 "type": "object"
2956 },
2957 "ParDoInstruction": {
2958 "description": "An instruction that does a ParDo operation. Takes on e main input and zero or more side inputs, and produces zero or more outputs. Ru ns user code.",
2959 "id": "ParDoInstruction",
2960 "properties": {
2961 "input": {
2962 "$ref": "InstructionInput",
2963 "description": "The input."
2964 },
2965 "sideInputs": {
2966 "description": "Zero or more side inputs.",
2967 "items": {
2968 "$ref": "SideInputInfo"
2969 },
2970 "type": "array"
2971 },
2972 "userFn": {
2973 "additionalProperties": {
2974 "description": "Properties of the object.",
2975 "type": "any"
2976 },
2977 "description": "The user function to invoke.",
2978 "type": "object"
2979 },
2980 "numOutputs": {
2981 "description": "The number of outputs.",
2982 "format": "int32",
2983 "type": "integer"
2984 },
2985 "multiOutputInfos": {
2986 "description": "Information about each of the outputs, if us er_fn is a MultiDoFn.",
2987 "items": {
2988 "$ref": "MultiOutputInfo"
2989 },
2990 "type": "array"
2991 }
2992 },
2993 "type": "object"
2994 },
2995 "SideInputInfo": {
2996 "description": "Information about a side input of a DoFn or an input of a SeqDoFn.",
2997 "id": "SideInputInfo",
2998 "properties": {
2999 "sources": {
3000 "description": "The source(s) to read element(s) from to get the value of this side input. If more than one source, then the elements are ta ken from the sources, in the specified order if order matters. At least one sour ce is required.",
3001 "items": {
3002 "$ref": "Source"
3003 },
3004 "type": "array"
3005 },
3006 "kind": {
3007 "additionalProperties": {
3008 "description": "Properties of the object.",
3009 "type": "any"
3010 },
3011 "description": "How to interpret the source element(s) as a side input value.",
3012 "type": "object"
3013 },
3014 "tag": {
3015 "description": "The id of the tag the user code will access this side input by; this should correspond to the tag of some MultiOutputInfo.",
3016 "type": "string"
3017 }
3018 },
3019 "type": "object"
3020 },
3021 "MultiOutputInfo": {
3022 "description": "Information about an output of a multi-output DoFn." ,
3023 "id": "MultiOutputInfo",
3024 "properties": {
3025 "tag": {
3026 "description": "The id of the tag the user code will emit to this output by; this should correspond to the tag of some SideInputInfo.",
3027 "type": "string"
3028 }
3029 },
3030 "type": "object"
3031 },
3032 "PartialGroupByKeyInstruction": {
3033 "description": "An instruction that does a partial group-by-key. One input and one output.",
3034 "id": "PartialGroupByKeyInstruction",
3035 "properties": {
3036 "input": {
3037 "$ref": "InstructionInput",
3038 "description": "Describes the input to the partial group-by- key instruction."
3039 },
3040 "inputElementCodec": {
3041 "additionalProperties": {
3042 "description": "Properties of the object.",
3043 "type": "any"
3044 },
3045 "description": "The codec to use for interpreting an element in the input PTable.",
3046 "type": "object"
3047 },
3048 "valueCombiningFn": {
3049 "additionalProperties": {
3050 "description": "Properties of the object.",
3051 "type": "any"
3052 },
3053 "description": "The value combining function to invoke.",
3054 "type": "object"
3055 },
3056 "sideInputs": {
3057 "description": "Zero or more side inputs.",
3058 "items": {
3059 "$ref": "SideInputInfo"
3060 },
3061 "type": "array"
3062 },
3063 "originalCombineValuesStepName": {
3064 "description": "If this instruction includes a combining fun ction, this is the name of the CombineValues instruction lifted into this instru ction.",
3065 "type": "string"
3066 },
3067 "originalCombineValuesInputStoreName": {
3068 "description": "If this instruction includes a combining fun ction this is the name of the intermediate store between the GBK and the Combine Values.",
3069 "type": "string"
3070 }
3071 },
3072 "type": "object"
3073 }, 3068 },
3074 "FlattenInstruction": { 3069 "FlattenInstruction": {
3075 "description": "An instruction that copies its inputs (zero or more) to its (single) output.", 3070 "description": "An instruction that copies its inputs (zero or more) to its (single) output.",
3076 "id": "FlattenInstruction", 3071 "id": "FlattenInstruction",
3077 "properties": { 3072 "properties": {
3078 "inputs": { 3073 "inputs": {
3079 "description": "Describes the inputs to the flatten instruct ion.", 3074 "description": "Describes the inputs to the flatten instruct ion.",
3080 "items": { 3075 "items": {
3081 "$ref": "InstructionInput" 3076 "$ref": "InstructionInput"
3082 }, 3077 },
3083 "type": "array" 3078 "type": "array"
3084 } 3079 }
3085 }, 3080 },
3086 "type": "object" 3081 "type": "object"
3087 }, 3082 },
3088 "InstructionOutput": { 3083 "PartialGroupByKeyInstruction": {
3089 "description": "An output of an instruction.", 3084 "description": "An instruction that does a partial group-by-key.\nOn e input and one output.",
3090 "id": "InstructionOutput", 3085 "id": "PartialGroupByKeyInstruction",
3091 "properties": { 3086 "properties": {
3092 "name": { 3087 "valueCombiningFn": {
3093 "description": "The user-provided name of this output.",
3094 "type": "string"
3095 },
3096 "systemName": {
3097 "description": "System-defined name of this output. Unique a cross the workflow.",
3098 "type": "string"
3099 },
3100 "originalName": {
3101 "description": "System-defined name for this output in the o riginal workflow graph. Outputs that do not contribute to an original instructio n do not set this.",
3102 "type": "string"
3103 },
3104 "codec": {
3105 "additionalProperties": { 3088 "additionalProperties": {
3106 "description": "Properties of the object.", 3089 "description": "Properties of the object.",
3107 "type": "any" 3090 "type": "any"
3108 }, 3091 },
3109 "description": "The codec to use to encode data being writte n via this output.", 3092 "description": "The value combining function to invoke.",
3110 "type": "object" 3093 "type": "object"
3111 }, 3094 },
3112 "onlyCountKeyBytes": { 3095 "inputElementCodec": {
3113 "description": "For system-generated byte and mean byte metr ics, certain instructions should only report the key size.",
3114 "type": "boolean"
3115 },
3116 "onlyCountValueBytes": {
3117 "description": "For system-generated byte and mean byte metr ics, certain instructions should only report the value size.",
3118 "type": "boolean"
3119 }
3120 },
3121 "type": "object"
3122 },
3123 "SeqMapTask": {
3124 "description": "Describes a particular function to invoke.",
3125 "id": "SeqMapTask",
3126 "properties": {
3127 "inputs": {
3128 "description": "Information about each of the inputs.",
3129 "items": {
3130 "$ref": "SideInputInfo"
3131 },
3132 "type": "array"
3133 },
3134 "userFn": {
3135 "additionalProperties": { 3096 "additionalProperties": {
3136 "description": "Properties of the object.", 3097 "description": "Properties of the object.",
3137 "type": "any" 3098 "type": "any"
3138 }, 3099 },
3139 "description": "The user function to invoke.", 3100 "description": "The codec to use for interpreting an element in the input PTable.",
3140 "type": "object" 3101 "type": "object"
3141 }, 3102 },
3142 "outputInfos": { 3103 "originalCombineValuesInputStoreName": {
3143 "description": "Information about each of the outputs.", 3104 "description": "If this instruction includes a combining fun ction this is the name of the\nintermediate store between the GBK and the Combin eValues.",
3144 "items": { 3105 "type": "string"
3145 "$ref": "SeqMapTaskOutputInfo" 3106 },
3146 }, 3107 "originalCombineValuesStepName": {
3147 "type": "array" 3108 "description": "If this instruction includes a combining fun ction, this is the name of the\nCombineValues instruction lifted into this instr uction.",
3148 }, 3109 "type": "string"
3149 "name": { 3110 },
3150 "description": "The user-provided name of the SeqDo operatio n.", 3111 "sideInputs": {
3151 "type": "string" 3112 "description": "Zero or more side inputs.",
3152 }, 3113 "items": {
3153 "systemName": { 3114 "$ref": "SideInputInfo"
3154 "description": "System-defined name of the SeqDo operation. Unique across the workflow.", 3115 },
3155 "type": "string" 3116 "type": "array"
3156 }, 3117 },
3157 "stageName": { 3118 "input": {
3158 "description": "System-defined name of the stage containing the SeqDo operation. Unique across the workflow.", 3119 "$ref": "InstructionInput",
3159 "type": "string" 3120 "description": "Describes the input to the partial group-by- key instruction."
3160 } 3121 }
3161 }, 3122 },
3162 "type": "object" 3123 "type": "object"
3163 }, 3124 },
3164 "SeqMapTaskOutputInfo": { 3125 "InstructionInput": {
3165 "description": "Information about an output of a SeqMapTask.", 3126 "description": "An input of an instruction, as a reference to an out put of a\nproducer instruction.",
3166 "id": "SeqMapTaskOutputInfo", 3127 "id": "InstructionInput",
3167 "properties": { 3128 "properties": {
3168 "tag": { 3129 "producerInstructionIndex": {
3169 "description": "The id of the TupleTag the user code will ta g the output value by.", 3130 "description": "The index (origin zero) of the parallel inst ruction that produces\nthe output to be consumed by this input. This index is r elative\nto the list of instructions in this input's instruction's\ncontaining M apTask.",
3170 "type": "string"
3171 },
3172 "sink": {
3173 "$ref": "Sink",
3174 "description": "The sink to write the output value to."
3175 }
3176 },
3177 "type": "object"
3178 },
3179 "ShellTask": {
3180 "description": "A task which consists of a shell command for the wor ker to execute.",
3181 "id": "ShellTask",
3182 "properties": {
3183 "command": {
3184 "description": "The shell command to run.",
3185 "type": "string"
3186 },
3187 "exitCode": {
3188 "description": "Exit code for the task.",
3189 "format": "int32", 3131 "format": "int32",
3190 "type": "integer" 3132 "type": "integer"
3191 } 3133 },
3192 }, 3134 "outputNum": {
3193 "type": "object" 3135 "description": "The output index (origin zero) within the pr oducer.",
3194 },
3195 "StreamingSetupTask": {
3196 "description": "A task which initializes part of a streaming Dataflo w job.",
3197 "id": "StreamingSetupTask",
3198 "properties": {
3199 "receiveWorkPort": {
3200 "description": "The TCP port on which the worker should list en for messages from other streaming computation workers.",
3201 "format": "int32", 3136 "format": "int32",
3202 "type": "integer" 3137 "type": "integer"
3203 }, 3138 }
3204 "workerHarnessPort": { 3139 },
3205 "description": "The TCP port used by the worker to communica te with the Dataflow worker harness.", 3140 "type": "object"
3141 },
3142 "StringList": {
3143 "description": "A metric value representing a list of strings.",
3144 "id": "StringList",
3145 "properties": {
3146 "elements": {
3147 "description": "Elements of the list.",
3148 "items": {
3149 "type": "string"
3150 },
3151 "type": "array"
3152 }
3153 },
3154 "type": "object"
3155 },
3156 "LeaseWorkItemRequest": {
3157 "description": "Request to lease WorkItems.",
3158 "id": "LeaseWorkItemRequest",
3159 "properties": {
3160 "workerCapabilities": {
3161 "description": "Worker capabilities. WorkItems might be limi ted to workers with specific\ncapabilities.",
3162 "items": {
3163 "type": "string"
3164 },
3165 "type": "array"
3166 },
3167 "workerId": {
3168 "description": "Identifies the worker leasing work -- typica lly the ID of the\nvirtual machine running the worker.",
3169 "type": "string"
3170 },
3171 "requestedLeaseDuration": {
3172 "description": "The initial lease period.",
3173 "format": "google-duration",
3174 "type": "string"
3175 },
3176 "currentWorkerTime": {
3177 "description": "The current timestamp at the worker.",
3178 "format": "google-datetime",
3179 "type": "string"
3180 },
3181 "workItemTypes": {
3182 "description": "Filter for WorkItem type.",
3183 "items": {
3184 "type": "string"
3185 },
3186 "type": "array"
3187 },
3188 "location": {
3189 "description": "The location which contains the WorkItem's j ob.",
3190 "type": "string"
3191 }
3192 },
3193 "type": "object"
3194 },
3195 "GetDebugConfigRequest": {
3196 "description": "Request to get updated debug configuration for compo nent.",
3197 "id": "GetDebugConfigRequest",
3198 "properties": {
3199 "componentId": {
3200 "description": "The internal component id for which debug co nfiguration is\nrequested.",
3201 "type": "string"
3202 },
3203 "workerId": {
3204 "description": "The worker id, i.e., VM hostname.",
3205 "type": "string"
3206 }
3207 },
3208 "type": "object"
3209 },
3210 "ReportWorkItemStatusRequest": {
3211 "description": "Request to report the status of WorkItems.",
3212 "id": "ReportWorkItemStatusRequest",
3213 "properties": {
3214 "workerId": {
3215 "description": "The ID of the worker reporting the WorkItem status. If this\ndoes not match the ID of the worker which the Dataflow service \nbelieves currently has the lease on the WorkItem, the report\nwill be dropped (with an error response).",
3216 "type": "string"
3217 },
3218 "currentWorkerTime": {
3219 "description": "The current timestamp at the worker.",
3220 "format": "google-datetime",
3221 "type": "string"
3222 },
3223 "location": {
3224 "description": "The location which contains the WorkItem's j ob.",
3225 "type": "string"
3226 },
3227 "workItemStatuses": {
3228 "description": "The order is unimportant, except that the or der of the\nWorkItemServiceState messages in the ReportWorkItemStatusResponse\nc orresponds to the order of WorkItemStatus messages here.",
3229 "items": {
3230 "$ref": "WorkItemStatus"
3231 },
3232 "type": "array"
3233 }
3234 },
3235 "type": "object"
3236 },
3237 "StreamingConfigTask": {
3238 "description": "A task that carries configuration information for st reaming computations.",
3239 "id": "StreamingConfigTask",
3240 "properties": {
3241 "userStepToStateFamilyNameMap": {
3242 "additionalProperties": {
3243 "type": "string"
3244 },
3245 "description": "Map from user step names to state families." ,
3246 "type": "object"
3247 },
3248 "streamingComputationConfigs": {
3249 "description": "Set of computation configuration information .",
3250 "items": {
3251 "$ref": "StreamingComputationConfig"
3252 },
3253 "type": "array"
3254 }
3255 },
3256 "type": "object"
3257 },
3258 "JobExecutionInfo": {
3259 "description": "Additional information about how a Cloud Dataflow jo b will be executed that\nisn't contained in the submitted job.",
3260 "id": "JobExecutionInfo",
3261 "properties": {
3262 "stages": {
3263 "additionalProperties": {
3264 "$ref": "JobExecutionStageInfo"
3265 },
3266 "description": "A mapping from each stage to the information about that stage.",
3267 "type": "object"
3268 }
3269 },
3270 "type": "object"
3271 },
3272 "Step": {
3273 "description": "Defines a particular step within a Cloud Dataflow jo b.\n\nA job consists of multiple steps, each of which performs some\nspecific op eration as part of the overall job. Data is typically\npassed from one step to another as part of the job.\n\nHere's an example of a sequence of steps which to gether implement a\nMap-Reduce job:\n\n * Read a collection of data from some s ource, parsing the\n collection's elements.\n\n * Validate the elements.\n\n * Apply a user-defined function to map each element to some value\n and ext ract an element-specific key value.\n\n * Group elements with the same key into a single element with\n that key, transforming a multiply-keyed collection i nto a\n uniquely-keyed collection.\n\n * Write the elements out to some data sink.\n\nNote that the Cloud Dataflow service may be used to run many different \ntypes of jobs, not just Map-Reduce.",
3274 "id": "Step",
3275 "properties": {
3276 "name": {
3277 "description": "The name that identifies the step. This must be unique for each\nstep with respect to all other steps in the Cloud Dataflow job.",
3278 "type": "string"
3279 },
3280 "kind": {
3281 "description": "The kind of step in the Cloud Dataflow job." ,
3282 "type": "string"
3283 },
3284 "properties": {
3285 "additionalProperties": {
3286 "description": "Properties of the object.",
3287 "type": "any"
3288 },
3289 "description": "Named properties associated with the step. E ach kind of\npredefined step has its own required set of properties.",
3290 "type": "object"
3291 }
3292 },
3293 "type": "object"
3294 },
3295 "FailedLocation": {
3296 "description": "Indicates which location failed to respond to a requ est for data.",
3297 "id": "FailedLocation",
3298 "properties": {
3299 "name": {
3300 "description": "The name of the failed location.",
3301 "type": "string"
3302 }
3303 },
3304 "type": "object"
3305 },
3306 "Disk": {
3307 "description": "Describes the data disk used by a workflow job.",
3308 "id": "Disk",
3309 "properties": {
3310 "sizeGb": {
3311 "description": "Size of disk in GB. If zero or unspecified, the service will\nattempt to choose a reasonable default.",
3206 "format": "int32", 3312 "format": "int32",
3207 "type": "integer" 3313 "type": "integer"
3208 }, 3314 },
3209 "streamingComputationTopology": { 3315 "diskType": {
3210 "$ref": "TopologyConfig", 3316 "description": "Disk storage type, as defined by Google Comp ute Engine. This\nmust be a disk type appropriate to the project and zone in wh ich\nthe workers will run. If unknown or unspecified, the service\nwill attempt to choose a reasonable default.\n\nFor example, the standard persistent disk ty pe is a resource name\ntypically ending in \"pd-standard\". If SSD persistent d isks are\navailable, the resource name typically ends with \"pd-ssd\". The\nact ual valid values are defined the Google Compute Engine API,\nnot by the Cloud Da taflow API; consult the Google Compute Engine\ndocumentation for more informatio n about determining the set of\navailable disk types for a particular project an d zone.\n\nGoogle Compute Engine Disk types are local to a particular\nproject i n a particular zone, and so the resource name will\ntypically look something lik e this:\n\ncompute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-st andard",
3211 "description": "The global topology of the streaming Dataflo w job." 3317 "type": "string"
3212 }, 3318 },
3213 "drain": { 3319 "mountPoint": {
3214 "description": "The user has requested drain.", 3320 "description": "Directory in a VM where disk is mounted.",
3215 "type": "boolean" 3321 "type": "string"
3216 } 3322 }
3217 }, 3323 },
3218 "type": "object" 3324 "type": "object"
3219 }, 3325 },
3220 "TopologyConfig": { 3326 "ListJobMessagesResponse": {
3221 "description": "Global topology of the streaming Dataflow job, inclu ding all computations and their sharded locations.", 3327 "description": "Response to a request to list job messages.",
3222 "id": "TopologyConfig", 3328 "id": "ListJobMessagesResponse",
3223 "properties": { 3329 "properties": {
3224 "computations": { 3330 "jobMessages": {
3225 "description": "The computations associated with a streaming Dataflow job.", 3331 "description": "Messages in ascending timestamp order.",
3226 "items": { 3332 "items": {
3227 "$ref": "ComputationTopology" 3333 "$ref": "JobMessage"
3228 }, 3334 },
3229 "type": "array" 3335 "type": "array"
3230 }, 3336 },
3231 "dataDiskAssignments": { 3337 "nextPageToken": {
3232 "description": "The disks assigned to a streaming Dataflow j ob.", 3338 "description": "The token to obtain the next page of results if there are more.",
3233 "items": { 3339 "type": "string"
3234 "$ref": "DataDiskAssignment" 3340 }
3235 }, 3341 },
3236 "type": "array" 3342 "type": "object"
3237 }, 3343 },
3238 "userStageToComputationNameMap": { 3344 "CounterMetadata": {
3239 "additionalProperties": { 3345 "description": "CounterMetadata includes all static non-name non-val ue counter attributes.",
3240 "type": "string" 3346 "id": "CounterMetadata",
3241 }, 3347 "properties": {
3242 "description": "Maps user stage names to stable computation names.", 3348 "standardUnits": {
3243 "type": "object" 3349 "description": "System defined Units, see above enum.",
3244 }, 3350 "enum": [
3245 "forwardingKeyBits": { 3351 "BYTES",
3246 "description": "The size (in bits) of keys that will be assi gned to source messages.", 3352 "BYTES_PER_SEC",
3247 "format": "int32", 3353 "MILLISECONDS",
3248 "type": "integer" 3354 "MICROSECONDS",
3249 }, 3355 "NANOSECONDS",
3250 "persistentStateVersion": { 3356 "TIMESTAMP_MSEC",
3251 "description": "Version number for persistent state.", 3357 "TIMESTAMP_USEC",
3252 "format": "int32", 3358 "TIMESTAMP_NSEC"
3253 "type": "integer" 3359 ],
3254 } 3360 "enumDescriptions": [
3255 }, 3361 "Counter returns a value in bytes.",
3256 "type": "object" 3362 "Counter returns a value in bytes per second.",
3257 }, 3363 "Counter returns a value in milliseconds.",
3258 "ComputationTopology": { 3364 "Counter returns a value in microseconds.",
3259 "description": "All configuration data for a particular Computation. ", 3365 "Counter returns a value in nanoseconds.",
3260 "id": "ComputationTopology", 3366 "Counter returns a timestamp in milliseconds.",
3261 "properties": { 3367 "Counter returns a timestamp in microseconds.",
3262 "systemStageName": { 3368 "Counter returns a timestamp in nanoseconds."
3263 "description": "The system stage name.", 3369 ],
3264 "type": "string" 3370 "type": "string"
3265 }, 3371 },
3266 "computationId": { 3372 "otherUnits": {
3267 "description": "The ID of the computation.", 3373 "description": "A string referring to the unit type.",
3268 "type": "string" 3374 "type": "string"
3269 }, 3375 },
3270 "userStageName": { 3376 "kind": {
3271 "description": "The user stage name.", 3377 "description": "Counter aggregation kind.",
3272 "type": "string" 3378 "enum": [
3273 }, 3379 "INVALID",
3274 "keyRanges": { 3380 "SUM",
3275 "description": "The key ranges processed by the computation. ", 3381 "MAX",
3276 "items": { 3382 "MIN",
3277 "$ref": "KeyRangeLocation" 3383 "MEAN",
3278 }, 3384 "OR",
3279 "type": "array" 3385 "AND",
3280 }, 3386 "SET",
3281 "inputs": { 3387 "DISTRIBUTION"
3282 "description": "The inputs to the computation.", 3388 ],
3283 "items": { 3389 "enumDescriptions": [
3284 "$ref": "StreamLocation" 3390 "Counter aggregation kind was not set.",
3285 }, 3391 "Aggregated value is the sum of all contributed values." ,
3286 "type": "array" 3392 "Aggregated value is the max of all contributed values." ,
3287 }, 3393 "Aggregated value is the min of all contributed values." ,
3288 "outputs": { 3394 "Aggregated value is the mean of all contributed values. ",
3289 "description": "The outputs from the computation.", 3395 "Aggregated value represents the logical 'or' of all con tributed values.",
3290 "items": { 3396 "Aggregated value represents the logical 'and' of all co ntributed values.",
3291 "$ref": "StreamLocation" 3397 "Aggregated value is a set of unique contributed values. ",
3292 }, 3398 "Aggregated value captures statistics about a distributi on."
3293 "type": "array" 3399 ],
3294 }, 3400 "type": "string"
3295 "stateFamilies": { 3401 },
3296 "description": "The state family values.", 3402 "description": {
3297 "items": { 3403 "description": "Human-readable description of the counter se mantics.",
3298 "$ref": "StateFamilyConfig" 3404 "type": "string"
3299 }, 3405 }
3300 "type": "array" 3406 },
3301 } 3407 "type": "object"
3302 }, 3408 },
3303 "type": "object" 3409 "ApproximateReportedProgress": {
3304 }, 3410 "description": "A progress measurement of a WorkItem by a worker.",
3305 "KeyRangeLocation": { 3411 "id": "ApproximateReportedProgress",
3306 "description": "Location information for a specific key-range of a s harded computation. Currently we only support UTF-8 character splits to simplify encoding into JSON.", 3412 "properties": {
3307 "id": "KeyRangeLocation", 3413 "remainingParallelism": {
3308 "properties": { 3414 "$ref": "ReportedParallelism",
3309 "start": { 3415 "description": "Total amount of parallelism in the input of this task that remains,\n(i.e. can be delegated to this task and any new tasks v ia dynamic\nsplitting). Always at least 1 for non-finished work items and 0 for\ nfinished.\n\n\"Amount of parallelism\" refers to how many non-empty parts of th e input\ncan be read in parallel. This does not necessarily equal number\nof rec ords. An input that can be read in parallel down to the\nindividual records is c alled \"perfectly splittable\".\nAn example of non-perfectly parallelizable inpu t is a block-compressed\nfile format where a block of records has to be read as a whole,\nbut different blocks can be read in parallel.\n\nExamples:\n* If we ar e processing record #30 (starting at 1) out of 50 in a perfectly\n splittable 5 0-record input, this value should be 21 (20 remaining + 1\n current).\n* If we are reading through block 3 in a block-compressed file consisting\n of 5 blocks , this value should be 3 (since blocks 4 and 5 can be\n processed in parallel b y new tasks via dynamic splitting and the current\n task remains processing blo ck 3).\n* If we are reading through the last block in a block-compressed file,\n or reading or processing the last record in a perfectly splittable\n input, t his value should be 1, because apart from the current task, no\n additional rem ainder can be split off."
3310 "description": "The start (inclusive) of the key range.", 3416 },
3311 "type": "string" 3417 "position": {
3312 }, 3418 "$ref": "Position",
3313 "end": { 3419 "description": "A Position within the work to represent a pr ogress."
3314 "description": "The end (exclusive) of the key range.", 3420 },
3315 "type": "string" 3421 "fractionConsumed": {
3316 }, 3422 "description": "Completion as fraction of the input consumed , from 0.0 (beginning, nothing\nconsumed), to 1.0 (end of the input, entire inpu t consumed).",
3317 "deliveryEndpoint": { 3423 "format": "double",
3318 "description": "The physical location of this range assignme nt to be used for streaming computation cross-worker message delivery.", 3424 "type": "number"
3319 "type": "string" 3425 },
3320 }, 3426 "consumedParallelism": {
3321 "persistentDirectory": { 3427 "$ref": "ReportedParallelism",
3322 "description": "The location of the persistent state for thi s range, as a persistent directory in the worker local filesystem.", 3428 "description": "Total amount of parallelism in the portion o f input of this task that has\nalready been consumed and is no longer active. In the first two examples\nabove (see remaining_parallelism), the value should be 29 or 2\nrespectively. The sum of remaining_parallelism and consumed_parallelis m\nshould equal the total amount of parallelism in this work item. If\nspecifie d, must be finite."
3323 "type": "string"
3324 },
3325 "dataDisk": {
3326 "description": "The name of the data disk where data for thi s range is stored. This name is local to the Google Cloud Platform project and u niquely identifies the disk within that project, for example \"myproject-1014-10 4817-4c2-harness-0-disk-1\".",
3327 "type": "string"
3328 }
3329 },
3330 "type": "object"
3331 },
3332 "StreamLocation": {
3333 "description": "Describes a stream of data, either as input to be pr ocessed or as output of a streaming Dataflow job.",
3334 "id": "StreamLocation",
3335 "properties": {
3336 "streamingStageLocation": {
3337 "$ref": "StreamingStageLocation",
3338 "description": "The stream is part of another computation wi thin the current streaming Dataflow job."
3339 },
3340 "pubsubLocation": {
3341 "$ref": "PubsubLocation",
3342 "description": "The stream is a pubsub stream."
3343 },
3344 "sideInputLocation": {
3345 "$ref": "StreamingSideInputLocation",
3346 "description": "The stream is a streaming side input."
3347 },
3348 "customSourceLocation": {
3349 "$ref": "CustomSourceLocation",
3350 "description": "The stream is a custom source."
3351 }
3352 },
3353 "type": "object"
3354 },
3355 "StreamingStageLocation": {
3356 "description": "Identifies the location of a streaming computation s tage, for stage-to-stage communication.",
3357 "id": "StreamingStageLocation",
3358 "properties": {
3359 "streamId": {
3360 "description": "Identifies the particular stream within the streaming Dataflow job.",
3361 "type": "string"
3362 }
3363 },
3364 "type": "object"
3365 },
3366 "PubsubLocation": {
3367 "description": "Identifies a pubsub location to use for transferring data into or out of a streaming Dataflow job.",
3368 "id": "PubsubLocation",
3369 "properties": {
3370 "topic": {
3371 "description": "A pubsub topic, in the form of \"pubsub.goog leapis.com/topics/\n/\"",
3372 "type": "string"
3373 },
3374 "subscription": {
3375 "description": "A pubsub subscription, in the form of \"pubs ub.googleapis.com/subscriptions/\n/\"",
3376 "type": "string"
3377 },
3378 "timestampLabel": {
3379 "description": "If set, contains a pubsub label from which t o extract record timestamps. If left empty, record timestamps will be generated upon arrival.",
3380 "type": "string"
3381 },
3382 "idLabel": {
3383 "description": "If set, contains a pubsub label from which t o extract record ids. If left empty, record deduplication will be strictly best effort.",
3384 "type": "string"
3385 },
3386 "dropLateData": {
3387 "description": "Indicates whether the pipeline allows late-a rriving data.",
3388 "type": "boolean"
3389 },
3390 "trackingSubscription": {
3391 "description": "If set, specifies the pubsub subscription th at will be used for tracking custom time timestamps for watermark estimation.",
3392 "type": "string"
3393 },
3394 "withAttributes": {
3395 "description": "If true, then the client has requested to ge t pubsub attributes.",
3396 "type": "boolean"
3397 }
3398 },
3399 "type": "object"
3400 },
3401 "StreamingSideInputLocation": {
3402 "description": "Identifies the location of a streaming side input.",
3403 "id": "StreamingSideInputLocation",
3404 "properties": {
3405 "tag": {
3406 "description": "Identifies the particular side input within the streaming Dataflow job.",
3407 "type": "string"
3408 },
3409 "stateFamily": {
3410 "description": "Identifies the state family where this side input is stored.",
3411 "type": "string"
3412 }
3413 },
3414 "type": "object"
3415 },
3416 "CustomSourceLocation": {
3417 "description": "Identifies the location of a custom souce.",
3418 "id": "CustomSourceLocation",
3419 "properties": {
3420 "stateful": {
3421 "description": "Whether this source is stateful.",
3422 "type": "boolean"
3423 } 3429 }
3424 }, 3430 },
3425 "type": "object" 3431 "type": "object"
3426 }, 3432 },
3427 "StateFamilyConfig": { 3433 "StateFamilyConfig": {
3428 "description": "State family configuration.", 3434 "description": "State family configuration.",
3429 "id": "StateFamilyConfig", 3435 "id": "StateFamilyConfig",
3430 "properties": { 3436 "properties": {
3431 "stateFamily": { 3437 "stateFamily": {
3432 "description": "The state family value.", 3438 "description": "The state family value.",
3433 "type": "string" 3439 "type": "string"
3434 }, 3440 },
3435 "isRead": { 3441 "isRead": {
3436 "description": "If true, this family corresponds to a read o peration.", 3442 "description": "If true, this family corresponds to a read o peration.",
3437 "type": "boolean" 3443 "type": "boolean"
3438 } 3444 }
3439 }, 3445 },
3440 "type": "object" 3446 "type": "object"
3441 }, 3447 },
3442 "DataDiskAssignment": { 3448 "IntegerList": {
3443 "description": "Data disk assignment for a given VM instance.", 3449 "description": "A metric value representing a list of integers.",
3444 "id": "DataDiskAssignment", 3450 "id": "IntegerList",
3445 "properties": { 3451 "properties": {
3446 "vmInstance": { 3452 "elements": {
3447 "description": "VM instance name the data disks mounted to, for example \"myproject-1014-104817-4c2-harness-0\".", 3453 "description": "Elements of the list.",
3448 "type": "string" 3454 "items": {
3449 }, 3455 "$ref": "SplitInt64"
3450 "dataDisks": { 3456 },
3451 "description": "Mounted data disks. The order is important a data disk's 0-based index in this list defines which persistent directory the d isk is mounted to, for example the list of { \"myproject-1014-104817-4c2-harness -0-disk-0\" }, { \"myproject-1014-104817-4c2-harness-0-disk-1\" }.", 3457 "type": "array"
3452 "items": { 3458 }
3453 "type": "string" 3459 },
3454 }, 3460 "type": "object"
3455 "type": "array" 3461 },
3456 } 3462 "ResourceUtilizationReportResponse": {
3457 }, 3463 "description": "Service-side response to WorkerMessage reporting res ource utilization.",
3458 "type": "object" 3464 "id": "ResourceUtilizationReportResponse",
3459 }, 3465 "properties": {},
3460 "SourceOperationRequest": { 3466 "type": "object"
3461 "description": "A work item that represents the different operations that can be performed on a user-defined Source specification.", 3467 },
3462 "id": "SourceOperationRequest", 3468 "SourceSplitResponse": {
3463 "properties": { 3469 "description": "The response to a SourceSplitRequest.",
3464 "split": { 3470 "id": "SourceSplitResponse",
3465 "$ref": "SourceSplitRequest", 3471 "properties": {
3466 "description": "Information about a request to split a sourc e." 3472 "outcome": {
3467 }, 3473 "description": "Indicates whether splitting happened and pro duced a list of bundles.\nIf this is USE_CURRENT_SOURCE_AS_IS, the current sourc e should\nbe processed \"as is\" without splitting. \"bundles\" is ignored in th is case.\nIf this is SPLITTING_HAPPENED, then \"bundles\" contains a list of\nbu ndles into which the source was split.",
3468 "getMetadata": {
3469 "$ref": "SourceGetMetadataRequest",
3470 "description": "Information about a request to get metadata about a source."
3471 }
3472 },
3473 "type": "object"
3474 },
3475 "SourceSplitRequest": {
3476 "description": "Represents the operation to split a high-level Sourc e specification into bundles (parts for parallel processing). At a high level, s plitting of a source into bundles happens as follows: SourceSplitRequest is appl ied to the source. If it returns SOURCE_SPLIT_OUTCOME_USE_CURRENT, no further sp litting happens and the source is used \"as is\". Otherwise, splitting is applie d recursively to each produced DerivedSource. As an optimization, for any Source , if its does_not_need_splitting is true, the framework assumes that splitting t his source would return SOURCE_SPLIT_OUTCOME_USE_CURRENT, and doesn't initiate a SourceSplitRequest. This applies both to the initial source being split and to bundles produced from it.",
3477 "id": "SourceSplitRequest",
3478 "properties": {
3479 "source": {
3480 "$ref": "Source",
3481 "description": "Specification of the source to be split."
3482 },
3483 "options": {
3484 "$ref": "SourceSplitOptions",
3485 "description": "Hints for tuning the splitting process."
3486 }
3487 },
3488 "type": "object"
3489 },
3490 "SourceSplitOptions": {
3491 "description": "Hints for splitting a Source into bundles (parts for parallel processing) using SourceSplitRequest.",
3492 "id": "SourceSplitOptions",
3493 "properties": {
3494 "desiredBundleSizeBytes": {
3495 "description": "The source should be split into a set of bun dles where the estimated size of each is approximately this many bytes.",
3496 "format": "int64",
3497 "type": "string"
3498 },
3499 "desiredShardSizeBytes": {
3500 "description": "DEPRECATED in favor of desired_bundle_size_b ytes.",
3501 "format": "int64",
3502 "type": "string"
3503 }
3504 },
3505 "type": "object"
3506 },
3507 "SourceGetMetadataRequest": {
3508 "description": "A request to compute the SourceMetadata of a Source. ",
3509 "id": "SourceGetMetadataRequest",
3510 "properties": {
3511 "source": {
3512 "$ref": "Source",
3513 "description": "Specification of the source whose metadata s hould be computed."
3514 }
3515 },
3516 "type": "object"
3517 },
3518 "StreamingComputationTask": {
3519 "description": "A task which describes what action should be perform ed for the specified streaming computation ranges.",
3520 "id": "StreamingComputationTask",
3521 "properties": {
3522 "taskType": {
3523 "description": "A type of streaming computation task.",
3524 "enum": [ 3474 "enum": [
3525 "STREAMING_COMPUTATION_TASK_UNKNOWN", 3475 "SOURCE_SPLIT_OUTCOME_UNKNOWN",
3526 "STREAMING_COMPUTATION_TASK_STOP", 3476 "SOURCE_SPLIT_OUTCOME_USE_CURRENT",
3527 "STREAMING_COMPUTATION_TASK_START" 3477 "SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED"
3528 ], 3478 ],
3529 "type": "string" 3479 "enumDescriptions": [
3530 }, 3480 "The source split outcome is unknown, or unspecified.",
3531 "dataDisks": { 3481 "The current source should be processed \"as is\" withou t splitting.",
3532 "description": "Describes the set of data disks this task sh ould apply to.", 3482 "Splitting produced a list of bundles."
3533 "items": { 3483 ],
3534 "$ref": "MountedDataDisk" 3484 "type": "string"
3535 }, 3485 },
3536 "type": "array" 3486 "bundles": {
3537 }, 3487 "description": "If outcome is SPLITTING_HAPPENED, then this is a list of bundles\ninto which the source was split. Otherwise this field is i gnored.\nThis list can be empty, which means the source represents an empty inpu t.",
3538 "computationRanges": { 3488 "items": {
3539 "description": "Contains ranges of a streaming computation t his task should apply to.", 3489 "$ref": "DerivedSource"
3540 "items": { 3490 },
3541 "$ref": "StreamingComputationRanges" 3491 "type": "array"
3542 }, 3492 },
3543 "type": "array" 3493 "shards": {
3544 } 3494 "description": "DEPRECATED in favor of bundles.",
3545 }, 3495 "items": {
3546 "type": "object" 3496 "$ref": "SourceSplitShard"
3547 }, 3497 },
3548 "MountedDataDisk": { 3498 "type": "array"
3549 "description": "Describes mounted data disk.", 3499 }
3550 "id": "MountedDataDisk", 3500 },
3551 "properties": { 3501 "type": "object"
3552 "dataDisk": { 3502 },
3553 "description": "The name of the data disk. This name is loca l to the Google Cloud Platform project and uniquely identifies the disk within t hat project, for example \"myproject-1014-104817-4c2-harness-0-disk-1\".", 3503 "ParallelInstruction": {
3554 "type": "string" 3504 "description": "Describes a particular operation comprising a MapTas k.",
3555 } 3505 "id": "ParallelInstruction",
3556 }, 3506 "properties": {
3557 "type": "object" 3507 "name": {
3558 }, 3508 "description": "User-provided name of this operation.",
3559 "StreamingComputationRanges": { 3509 "type": "string"
3560 "description": "Describes full or partial data disk assignment infor mation of the computation ranges.", 3510 },
3561 "id": "StreamingComputationRanges", 3511 "parDo": {
3562 "properties": { 3512 "$ref": "ParDoInstruction",
3563 "computationId": { 3513 "description": "Additional information for ParDo instruction s."
3564 "description": "The ID of the computation.", 3514 },
3565 "type": "string" 3515 "read": {
3566 }, 3516 "$ref": "ReadInstruction",
3567 "rangeAssignments": { 3517 "description": "Additional information for Read instructions ."
3568 "description": "Data disk assignments for ranges from this c omputation.", 3518 },
3569 "items": { 3519 "flatten": {
3570 "$ref": "KeyRangeDataDiskAssignment" 3520 "$ref": "FlattenInstruction",
3571 }, 3521 "description": "Additional information for Flatten instructi ons."
3572 "type": "array" 3522 },
3573 } 3523 "originalName": {
3524 "description": "System-defined name for the operation in the original workflow graph.",
3525 "type": "string"
3526 },
3527 "write": {
3528 "$ref": "WriteInstruction",
3529 "description": "Additional information for Write instruction s."
3530 },
3531 "systemName": {
3532 "description": "System-defined name of this operation.\nUniq ue across the workflow.",
3533 "type": "string"
3534 },
3535 "partialGroupByKey": {
3536 "$ref": "PartialGroupByKeyInstruction",
3537 "description": "Additional information for PartialGroupByKey instructions."
3538 },
3539 "outputs": {
3540 "description": "Describes the outputs of the instruction.",
3541 "items": {
3542 "$ref": "InstructionOutput"
3543 },
3544 "type": "array"
3545 }
3574 }, 3546 },
3575 "type": "object" 3547 "type": "object"
3576 }, 3548 },
3577 "KeyRangeDataDiskAssignment": { 3549 "KeyRangeDataDiskAssignment": {
3578 "description": "Data disk assignment information for a specific key- range of a sharded computation. Currently we only support UTF-8 character splits to simplify encoding into JSON.", 3550 "description": "Data disk assignment information for a specific key- range of a sharded\ncomputation.\nCurrently we only support UTF-8 character spli ts to simplify encoding into\nJSON.",
3579 "id": "KeyRangeDataDiskAssignment", 3551 "id": "KeyRangeDataDiskAssignment",
3580 "properties": { 3552 "properties": {
3581 "start": { 3553 "start": {
3582 "description": "The start (inclusive) of the key range.", 3554 "description": "The start (inclusive) of the key range.",
3583 "type": "string" 3555 "type": "string"
3584 }, 3556 },
3557 "dataDisk": {
3558 "description": "The name of the data disk where data for thi s range is stored.\nThis name is local to the Google Cloud Platform project and uniquely\nidentifies the disk within that project, for example\n\"myproject-1014 -104817-4c2-harness-0-disk-1\".",
3559 "type": "string"
3560 },
3585 "end": { 3561 "end": {
3586 "description": "The end (exclusive) of the key range.", 3562 "description": "The end (exclusive) of the key range.",
3587 "type": "string" 3563 "type": "string"
3588 }, 3564 }
3589 "dataDisk": { 3565 },
3590 "description": "The name of the data disk where data for thi s range is stored. This name is local to the Google Cloud Platform project and u niquely identifies the disk within that project, for example \"myproject-1014-10 4817-4c2-harness-0-disk-1\".", 3566 "type": "object"
3591 "type": "string" 3567 },
3592 } 3568 "Package": {
3593 }, 3569 "description": "The packages that must be installed in order for a w orker to run the\nsteps of the Cloud Dataflow job that will be assigned to its w orker\npool.\n\nThis is the mechanism by which the Cloud Dataflow SDK causes cod e to\nbe loaded onto the workers. For example, the Cloud Dataflow Java SDK\nmigh t use this to install jars containing the user's code and all of the\nvarious de pendencies (libraries, data files, etc.) required in order\nfor that code to run .",
3594 "type": "object" 3570 "id": "Package",
3595 }, 3571 "properties": {
3596 "StreamingConfigTask": { 3572 "location": {
3597 "description": "A task that carries configuration information for st reaming computations.", 3573 "description": "The resource to read the package from. The s upported resource type is:\n\nGoogle Cloud Storage:\n\n storage.googleapis.com/ {bucket}\n bucket.storage.googleapis.com/",
3598 "id": "StreamingConfigTask", 3574 "type": "string"
3599 "properties": { 3575 },
3600 "streamingComputationConfigs": { 3576 "name": {
3601 "description": "Set of computation configuration information .", 3577 "description": "The name of the package.",
3578 "type": "string"
3579 }
3580 },
3581 "type": "object"
3582 },
3583 "ParDoInstruction": {
3584 "description": "An instruction that does a ParDo operation.\nTakes o ne main input and zero or more side inputs, and produces\nzero or more outputs.\ nRuns user code.",
3585 "id": "ParDoInstruction",
3586 "properties": {
3587 "numOutputs": {
3588 "description": "The number of outputs.",
3589 "format": "int32",
3590 "type": "integer"
3591 },
3592 "sideInputs": {
3593 "description": "Zero or more side inputs.",
3602 "items": { 3594 "items": {
3603 "$ref": "StreamingComputationConfig" 3595 "$ref": "SideInputInfo"
3604 }, 3596 },
3605 "type": "array" 3597 "type": "array"
3606 }, 3598 },
3607 "userStepToStateFamilyNameMap": { 3599 "multiOutputInfos": {
3608 "additionalProperties": { 3600 "description": "Information about each of the outputs, if us er_fn is a MultiDoFn.",
3609 "type": "string"
3610 },
3611 "description": "Map from user step names to state families." ,
3612 "type": "object"
3613 }
3614 },
3615 "type": "object"
3616 },
3617 "StreamingComputationConfig": {
3618 "description": "Configuration information for a single streaming com putation.",
3619 "id": "StreamingComputationConfig",
3620 "properties": {
3621 "computationId": {
3622 "description": "Unique identifier for this computation.",
3623 "type": "string"
3624 },
3625 "systemName": {
3626 "description": "System defined name for this computation.",
3627 "type": "string"
3628 },
3629 "stageName": {
3630 "description": "Stage name of this computation.",
3631 "type": "string"
3632 },
3633 "instructions": {
3634 "description": "Instructions that comprise the computation." ,
3635 "items": { 3601 "items": {
3636 "$ref": "ParallelInstruction" 3602 "$ref": "MultiOutputInfo"
3637 }, 3603 },
3638 "type": "array" 3604 "type": "array"
3639 } 3605 },
3640 }, 3606 "userFn": {
3641 "type": "object" 3607 "additionalProperties": {
3642 }, 3608 "description": "Properties of the object.",
3643 "SendWorkerMessagesRequest": { 3609 "type": "any"
3644 "description": "A request for sending worker messages to the service .",
3645 "id": "SendWorkerMessagesRequest",
3646 "properties": {
3647 "workerMessages": {
3648 "description": "The WorkerMessages to send.",
3649 "items": {
3650 "$ref": "WorkerMessage"
3651 }, 3610 },
3652 "type": "array" 3611 "description": "The user function to invoke.",
3653 } 3612 "type": "object"
3654 }, 3613 },
3655 "type": "object" 3614 "input": {
3615 "$ref": "InstructionInput",
3616 "description": "The input."
3617 }
3618 },
3619 "type": "object"
3620 },
3621 "MetricUpdate": {
3622 "description": "Describes the state of a metric.",
3623 "id": "MetricUpdate",
3624 "properties": {
3625 "meanSum": {
3626 "description": "Worker-computed aggregate value for the \"Me an\" aggregation kind.\nThis holds the sum of the aggregated values and is used in combination\nwith mean_count below to obtain the actual mean aggregate value. \nThe only possible value types are Long and Double.",
3627 "type": "any"
3628 },
3629 "updateTime": {
3630 "description": "Timestamp associated with the metric value. Optional when workers are\nreporting work progress; it will be filled in respons es from the\nmetrics API.",
3631 "format": "google-datetime",
3632 "type": "string"
3633 },
3634 "name": {
3635 "$ref": "MetricStructuredName",
3636 "description": "Name of the metric."
3637 },
3638 "set": {
3639 "description": "Worker-computed aggregate value for the \"Se t\" aggregation kind. The only\npossible value type is a list of Values whose t ype can be Long, Double,\nor String, according to the metric's type. All Values in the list must\nbe of the same type.",
3640 "type": "any"
3641 },
3642 "internal": {
3643 "description": "Worker-computed aggregate value for internal use by the Dataflow\nservice.",
3644 "type": "any"
3645 },
3646 "cumulative": {
3647 "description": "True if this metric is reported as the total cumulative aggregate\nvalue accumulated since the worker started working on thi s WorkItem.\nBy default this is false, indicating that this metric is reported\n as a delta that is not associated with any WorkItem.",
3648 "type": "boolean"
3649 },
3650 "kind": {
3651 "description": "Metric aggregation kind. The possible metri c aggregation kinds are\n\"Sum\", \"Max\", \"Min\", \"Mean\", \"Set\", \"And\", and \"Or\".\nThe specified aggregation kind is case-insensitive.\n\nIf omitted, this is not an aggregated value but instead\na single metric sample value.",
3652 "type": "string"
3653 },
3654 "scalar": {
3655 "description": "Worker-computed aggregate value for aggregat ion kinds \"Sum\", \"Max\", \"Min\",\n\"And\", and \"Or\". The possible value t ypes are Long, Double, and Boolean.",
3656 "type": "any"
3657 },
3658 "meanCount": {
3659 "description": "Worker-computed aggregate value for the \"Me an\" aggregation kind.\nThis holds the count of the aggregated values and is use d in combination\nwith mean_sum above to obtain the actual mean aggregate value. \nThe only possible value type is Long.",
3660 "type": "any"
3661 }
3662 },
3663 "type": "object"
3664 },
3665 "CounterStructuredName": {
3666 "description": "Identifies a counter within a per-job namespace. Cou nters whose structured\nnames are the same get merged into a single value for th e job.",
3667 "id": "CounterStructuredName",
3668 "properties": {
3669 "workerId": {
3670 "description": "ID of a particular worker.",
3671 "type": "string"
3672 },
3673 "originNamespace": {
3674 "description": "A string containing a more specific namespac e of the counter's origin.",
3675 "type": "string"
3676 },
3677 "origin": {
3678 "description": "One of the standard Origins defined above.",
3679 "enum": [
3680 "SYSTEM",
3681 "USER"
3682 ],
3683 "enumDescriptions": [
3684 "Counter was created by the Dataflow system.",
3685 "Counter was created by the user."
3686 ],
3687 "type": "string"
3688 },
3689 "name": {
3690 "description": "Counter name. Not necessarily globally-uniqu e, but unique within the\ncontext of the other fields.\nRequired.",
3691 "type": "string"
3692 },
3693 "executionStepName": {
3694 "description": "Name of the stage. An execution step contain s multiple component steps.",
3695 "type": "string"
3696 },
3697 "componentStepName": {
3698 "description": "Name of the optimized step being executed by the workers.",
3699 "type": "string"
3700 },
3701 "portion": {
3702 "description": "Portion of this counter, either key or value .",
3703 "enum": [
3704 "ALL",
3705 "KEY",
3706 "VALUE"
3707 ],
3708 "enumDescriptions": [
3709 "Counter portion has not been set.",
3710 "Counter reports a key.",
3711 "Counter reports a value."
3712 ],
3713 "type": "string"
3714 },
3715 "originalStepName": {
3716 "description": "System generated name of the original step i n the user's graph, before\noptimization.",
3717 "type": "string"
3718 }
3719 },
3720 "type": "object"
3721 },
3722 "ApproximateProgress": {
3723 "description": "Obsolete in favor of ApproximateReportedProgress and ApproximateSplitRequest.",
3724 "id": "ApproximateProgress",
3725 "properties": {
3726 "percentComplete": {
3727 "description": "Obsolete.",
3728 "format": "float",
3729 "type": "number"
3730 },
3731 "remainingTime": {
3732 "description": "Obsolete.",
3733 "format": "google-duration",
3734 "type": "string"
3735 },
3736 "position": {
3737 "$ref": "Position",
3738 "description": "Obsolete."
3739 }
3740 },
3741 "type": "object"
3742 },
3743 "WorkerMessageResponse": {
3744 "description": "A worker_message response allows the server to pass information to the\nsender.",
3745 "id": "WorkerMessageResponse",
3746 "properties": {
3747 "workerMetricsResponse": {
3748 "$ref": "ResourceUtilizationReportResponse",
3749 "description": "Service's response to reporting worker metri cs (currently empty)."
3750 },
3751 "workerHealthReportResponse": {
3752 "$ref": "WorkerHealthReportResponse",
3753 "description": "The service's response to a worker's health report."
3754 }
3755 },
3756 "type": "object"
3656 }, 3757 },
3657 "WorkerMessage": { 3758 "WorkerMessage": {
3658 "description": "WorkerMessage provides information to the backend ab out a worker.", 3759 "description": "WorkerMessage provides information to the backend ab out a worker.",
3659 "id": "WorkerMessage", 3760 "id": "WorkerMessage",
3660 "properties": { 3761 "properties": {
3762 "workerMessageCode": {
3763 "$ref": "WorkerMessageCode",
3764 "description": "A worker message code."
3765 },
3766 "workerMetrics": {
3767 "$ref": "ResourceUtilizationReport",
3768 "description": "Resource metrics reported by workers."
3769 },
3661 "labels": { 3770 "labels": {
3662 "additionalProperties": { 3771 "additionalProperties": {
3663 "type": "string" 3772 "type": "string"
3664 }, 3773 },
3665 "description": "Labels are used to group WorkerMessages. For example, a worker_message about a particular container might have the labels: { \"JOB_ID\": \"2015-04-22\", \"WORKER_ID\": \"wordcount-vm-2015…\" \"CONTAINER_T YPE\": \"worker\", \"CONTAINER_ID\": \"ac1234def\"} Label tags typically corresp ond to Label enum values. However, for ease of development other strings can be used as tags. LABEL_UNSPECIFIED should not be used here.", 3774 "description": "Labels are used to group WorkerMessages.\nFo r example, a worker_message about a particular container\nmight have the labels: \n{ \"JOB_ID\": \"2015-04-22\",\n \"WORKER_ID\": \"wordcount-vm-2015…\"\n \"CO NTAINER_TYPE\": \"worker\",\n \"CONTAINER_ID\": \"ac1234def\"}\nLabel tags typi cally correspond to Label enum values. However, for ease\nof development other s trings can be used as tags. LABEL_UNSPECIFIED should\nnot be used here.",
3666 "type": "object" 3775 "type": "object"
3667 }, 3776 },
3668 "time": { 3777 "time": {
3669 "description": "The timestamp of the worker_message.", 3778 "description": "The timestamp of the worker_message.",
3779 "format": "google-datetime",
3670 "type": "string" 3780 "type": "string"
3671 }, 3781 },
3672 "workerHealthReport": { 3782 "workerHealthReport": {
3673 "$ref": "WorkerHealthReport", 3783 "$ref": "WorkerHealthReport",
3674 "description": "The health of a worker." 3784 "description": "The health of a worker."
3675 }, 3785 }
3676 "workerMessageCode": { 3786 },
3677 "$ref": "WorkerMessageCode", 3787 "type": "object"
3678 "description": "A worker message code." 3788 },
3679 }, 3789 "JobMetrics": {
3680 "workerMetrics": { 3790 "description": "JobMetrics contains a collection of metrics descibin g the detailed progress\nof a Dataflow job. Metrics correspond to user-defined a nd system-defined\nmetrics in the job.\n\nThis resource captures only the most r ecent values of each metric;\ntime-series data can be queried for them (under th e same metric names)\nfrom Cloud Monitoring.",
3681 "$ref": "ResourceUtilizationReport", 3791 "id": "JobMetrics",
3682 "description": "Resource metrics reported by workers." 3792 "properties": {
3683 } 3793 "metricTime": {
3684 }, 3794 "description": "Timestamp as of which metric values are curr ent.",
3685 "type": "object" 3795 "format": "google-datetime",
3686 }, 3796 "type": "string"
3687 "WorkerHealthReport": { 3797 },
3688 "description": "WorkerHealthReport contains information about the he alth of a worker. The VM should be identified by the labels attached to the Work erMessage that this health ping belongs to.", 3798 "metrics": {
3689 "id": "WorkerHealthReport", 3799 "description": "All metrics for this job.",
3690 "properties": {
3691 "vmIsHealthy": {
3692 "description": "Whether the VM is healthy.",
3693 "type": "boolean"
3694 },
3695 "vmStartupTime": {
3696 "description": "The time the VM was booted.",
3697 "type": "string"
3698 },
3699 "reportInterval": {
3700 "description": "The interval at which the worker is sending health reports. The default value of 0 should be interpreted as the field is not being explicitly set by the worker.",
3701 "type": "string"
3702 },
3703 "pods": {
3704 "description": "The pods running on the worker. See: http:// kubernetes.io/v1.1/docs/api-reference/v1/definitions.html#_v1_pod This field is used by the worker to send the status of the indvidual containers running on eac h worker.",
3705 "items": { 3800 "items": {
3706 "additionalProperties": { 3801 "$ref": "MetricUpdate"
3707 "description": "Properties of the object.",
3708 "type": "any"
3709 },
3710 "type": "object"
3711 }, 3802 },
3712 "type": "array" 3803 "type": "array"
3713 } 3804 }
3714 }, 3805 },
3715 "type": "object" 3806 "type": "object"
3716 }, 3807 },
3717 "WorkerMessageCode": { 3808 "FloatingPointList": {
3718 "description": "A message code is used to report status and error me ssages to the service. The message codes are intended to be machine readable. Th e service will take care of translating these into user understandable messages if necessary. Example use cases: 1. Worker processes reporting successful startu p. 2. Worker processes reporting specific errors (e.g. package staging failure). ", 3809 "description": "A metric value representing a list of floating point numbers.",
3719 "id": "WorkerMessageCode", 3810 "id": "FloatingPointList",
3720 "properties": { 3811 "properties": {
3721 "code": { 3812 "elements": {
3722 "description": "The code is a string intended for consumptio n by a machine that identifies the type of message being sent. Examples: 1. \"HA RNESS_STARTED\" might be used to indicate the worker harness has started. 2. \"G CS_DOWNLOAD_ERROR\" might be used to indicate an error downloading a GCS file as part of the boot process of one of the worker containers. This is a string and not an enum to make it easy to add new codes without waiting for an API change." , 3813 "description": "Elements of the list.",
3723 "type": "string"
3724 },
3725 "parameters": {
3726 "additionalProperties": {
3727 "description": "Properties of the object.",
3728 "type": "any"
3729 },
3730 "description": "Parameters contains specific information abo ut the code. This is a struct to allow parameters of different types. Examples: 1. For a \"HARNESS_STARTED\" message parameters might provide the name of the wo rker and additional data like timing information. 2. For a \"GCS_DOWNLOAD_ERROR\ " parameters might contain fields listing the GCS objects being downloaded and f ields containing errors. In general complex data structures should be avoided. I f a worker needs to send a specific and complicated data structure then please c onsider defining a new proto and adding it to the data oneof in WorkerMessageRes ponse. Conventions: Parameters should only be used for information that isn't ty pically passed as a label. hostname and other worker identifiers should almost a lways be passed as labels since they will be included on most messages.",
3731 "type": "object"
3732 }
3733 },
3734 "type": "object"
3735 },
3736 "ResourceUtilizationReport": {
3737 "description": "Worker metrics exported from workers. This contains resource utilization metrics accumulated from a variety of sources. For more inf ormation, see go/df-resource-signals. Note that this proto closely follows the s tructure of its DFE siblings in its contents.",
3738 "id": "ResourceUtilizationReport",
3739 "properties": {
3740 "metrics": {
3741 "description": "Each Struct must parallel DFE worker metrics protos (eg., cpu_time metric will have nested values “timestamp_ms, total_ms, r ate”).",
3742 "items": { 3814 "items": {
3743 "additionalProperties": { 3815 "format": "double",
3744 "description": "Properties of the object.", 3816 "type": "number"
3745 "type": "any"
3746 },
3747 "type": "object"
3748 }, 3817 },
3749 "type": "array" 3818 "type": "array"
3750 } 3819 }
3751 }, 3820 },
3752 "type": "object" 3821 "type": "object"
3753 }, 3822 },
3754 "SendWorkerMessagesResponse": { 3823 "CounterUpdate": {
3755 "description": "The response to the worker messages.", 3824 "description": "An update to a Counter sent from a worker.",
3756 "id": "SendWorkerMessagesResponse", 3825 "id": "CounterUpdate",
3757 "properties": { 3826 "properties": {
3758 "workerMessageResponses": { 3827 "shortId": {
3759 "description": "The servers response to the worker messages. ", 3828 "description": "The service-generated short identifier for t his counter.\nThe short_id -> (name, metadata) mapping is constant for the lifet ime of\na job.",
3760 "items": { 3829 "format": "int64",
3761 "$ref": "WorkerMessageResponse" 3830 "type": "string"
3762 }, 3831 },
3763 "type": "array" 3832 "floatingPointList": {
3764 } 3833 "$ref": "FloatingPointList",
3765 }, 3834 "description": "List of floating point numbers, for Set."
3766 "type": "object" 3835 },
3767 }, 3836 "integer": {
3768 "WorkerMessageResponse": { 3837 "$ref": "SplitInt64",
3769 "description": "A worker_message response allows the server to pass information to the sender.", 3838 "description": "Integer value for Sum, Max, Min."
3770 "id": "WorkerMessageResponse", 3839 },
3771 "properties": { 3840 "structuredNameAndMetadata": {
3772 "workerHealthReportResponse": { 3841 "$ref": "CounterStructuredNameAndMetadata",
3773 "$ref": "WorkerHealthReportResponse", 3842 "description": "Counter structured name and metadata."
3774 "description": "The service's response to a worker's health report." 3843 },
3775 }, 3844 "integerList": {
3776 "workerMetricsResponse": { 3845 "$ref": "IntegerList",
3777 "$ref": "ResourceUtilizationReportResponse", 3846 "description": "List of integers, for Set."
3778 "description": "Service's response to reporting worker metri cs (currently empty)." 3847 },
3848 "floatingPoint": {
3849 "description": "Floating point value for Sum, Max, Min.",
3850 "format": "double",
3851 "type": "number"
3852 },
3853 "integerMean": {
3854 "$ref": "IntegerMean",
3855 "description": "Integer mean aggregation value for Mean."
3856 },
3857 "cumulative": {
3858 "description": "True if this counter is reported as the tota l cumulative aggregate\nvalue accumulated since the worker started working on th is WorkItem.\nBy default this is false, indicating that this counter is reported \nas a delta.",
3859 "type": "boolean"
3860 },
3861 "internal": {
3862 "description": "Value for internally-defined counters used b y the Dataflow service.",
3863 "type": "any"
3864 },
3865 "floatingPointMean": {
3866 "$ref": "FloatingPointMean",
3867 "description": "Floating point mean aggregation value for Me an."
3868 },
3869 "boolean": {
3870 "description": "Boolean value for And, Or.",
3871 "type": "boolean"
3872 },
3873 "nameAndKind": {
3874 "$ref": "NameAndKind",
3875 "description": "Counter name and aggregation type."
3876 },
3877 "stringList": {
3878 "$ref": "StringList",
3879 "description": "List of strings, for Set."
3880 },
3881 "distribution": {
3882 "$ref": "DistributionUpdate",
3883 "description": "Distribution data"
3884 }
3885 },
3886 "type": "object"
3887 },
3888 "SourceMetadata": {
3889 "description": "Metadata about a Source useful for automatically opt imizing\nand tuning the pipeline, etc.",
3890 "id": "SourceMetadata",
3891 "properties": {
3892 "producesSortedKeys": {
3893 "description": "Whether this source is known to produce key/ value pairs with\nthe (encoded) keys in lexicographically sorted order.",
3894 "type": "boolean"
3895 },
3896 "infinite": {
3897 "description": "Specifies that the size of this source is kn own to be infinite\n(this is a streaming source).",
3898 "type": "boolean"
3899 },
3900 "estimatedSizeBytes": {
3901 "description": "An estimate of the total size (in bytes) of the data that would be\nread from this source. This estimate is in terms of ext ernal storage\nsize, before any decompression or other processing done by the re ader.",
3902 "format": "int64",
3903 "type": "string"
3904 }
3905 },
3906 "type": "object"
3907 },
3908 "DistributionUpdate": {
3909 "description": "A metric value representing a distribution.",
3910 "id": "DistributionUpdate",
3911 "properties": {
3912 "count": {
3913 "$ref": "SplitInt64",
3914 "description": "The count of the number of elements present in the distribution."
3915 },
3916 "min": {
3917 "$ref": "SplitInt64",
3918 "description": "The minimum value present in the distributio n."
3919 },
3920 "sumOfSquares": {
3921 "description": "Use a double since the sum of squares is lik ely to overflow int64.",
3922 "format": "double",
3923 "type": "number"
3924 },
3925 "max": {
3926 "$ref": "SplitInt64",
3927 "description": "The maximum value present in the distributio n."
3928 },
3929 "sum": {
3930 "$ref": "SplitInt64",
3931 "description": "Use an int64 since we'd prefer the added pre cision. If overflow is a common\nproblem we can detect it and use an additional int64 or a double."
3932 }
3933 },
3934 "type": "object"
3935 },
3936 "SourceFork": {
3937 "description": "DEPRECATED in favor of DynamicSourceSplit.",
3938 "id": "SourceFork",
3939 "properties": {
3940 "residualSource": {
3941 "$ref": "DerivedSource",
3942 "description": "DEPRECATED"
3943 },
3944 "primary": {
3945 "$ref": "SourceSplitShard",
3946 "description": "DEPRECATED"
3947 },
3948 "primarySource": {
3949 "$ref": "DerivedSource",
3950 "description": "DEPRECATED"
3951 },
3952 "residual": {
3953 "$ref": "SourceSplitShard",
3954 "description": "DEPRECATED"
3779 } 3955 }
3780 }, 3956 },
3781 "type": "object" 3957 "type": "object"
3782 }, 3958 },
3783 "WorkerHealthReportResponse": { 3959 "WorkerHealthReportResponse": {
3784 "description": "WorkerHealthReportResponse contains information retu rned to the worker in response to a health ping.", 3960 "description": "WorkerHealthReportResponse contains information retu rned to the worker\nin response to a health ping.",
3785 "id": "WorkerHealthReportResponse", 3961 "id": "WorkerHealthReportResponse",
3786 "properties": { 3962 "properties": {
3787 "reportInterval": { 3963 "reportInterval": {
3788 "description": "A positive value indicates the worker should change its reporting interval to the specified value. The default value of zero means no change in report rate is requested by the server.", 3964 "description": "A positive value indicates the worker should change its reporting interval\nto the specified value.\n\nThe default value of zero means no change in report rate is requested by\nthe server.",
3789 "type": "string" 3965 "format": "google-duration",
3790 } 3966 "type": "string"
3791 }, 3967 }
3792 "type": "object" 3968 },
3793 },
3794 "ResourceUtilizationReportResponse": {
3795 "description": "Service-side response to WorkerMessage reporting res ource utilization.",
3796 "id": "ResourceUtilizationReportResponse",
3797 "type": "object" 3969 "type": "object"
3798 } 3970 }
3799 }, 3971 },
3800 "servicePath": "", 3972 "servicePath": "",
3801 "title": "Google Dataflow API", 3973 "title": "Google Dataflow API",
3802 "version": "v1b3" 3974 "version": "v1b3"
3803 } 3975 }
OLDNEW
« no previous file with comments | « discovery/googleapis_beta/cloudresourcemanager__v1beta1.json ('k') | discovery/googleapis_beta/dataproc__v1beta1.json » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698