| OLD | NEW |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import logging | 5 import logging |
| 6 import os | 6 import os |
| 7 import socket | 7 import socket |
| 8 | 8 |
| 9 import infra_libs | 9 import infra_libs |
| 10 from infra_libs.event_mon.protos.chrome_infra_log_pb2 import ( | 10 from infra_libs.event_mon.protos.chrome_infra_log_pb2 import ( |
| 11 ChromeInfraEvent, ServiceEvent) | 11 ChromeInfraEvent, ServiceEvent) |
| 12 from infra_libs.event_mon import router as ev_router | 12 from infra_libs.event_mon import router as ev_router |
| 13 | 13 |
| 14 DEFAULT_SERVICE_ACCOUNT_CREDS = 'service-account-event-mon.json' | |
| 15 RUNTYPES = set(('dry', 'test', 'prod', 'file')) | 14 RUNTYPES = set(('dry', 'test', 'prod', 'file')) |
| 16 | 15 |
| 17 # Remote endpoints | 16 # Remote endpoints |
| 18 ENDPOINTS = { | 17 ENDPOINTS = { |
| 19 'test': 'https://jmt17.google.com/log', | 18 'test': 'https://jmt17.google.com/log', |
| 20 'prod': 'https://play.googleapis.com/log', | 19 'prod': 'https://play.googleapis.com/log', |
| 21 } | 20 } |
| 22 | 21 |
| 23 # Instance of router._Router (singleton) | 22 # Instance of router._Router (singleton) |
| 24 _router = None | 23 _router = None |
| 25 | 24 |
| 26 # Cache some generally useful values / options | 25 # Cache some generally useful values / options |
| 27 _cache = {} | 26 _cache = {} |
| 28 | 27 |
| 29 | 28 |
| 30 def add_argparse_options(parser): | 29 def add_argparse_options(parser): |
| 31 # The default values should make sense for local testing, not production. | 30 # The default values should make sense for local testing, not production. |
| 32 group = parser.add_argument_group('Event monitoring (event_mon) ' | 31 group = parser.add_argument_group('Event monitoring (event_mon) ' |
| 33 'global options') | 32 'global options') |
| 34 group.add_argument('--dry-run', default=False, | 33 group.add_argument('--dry-run', default=False, |
| 35 action='store_true', | 34 action='store_true', |
| 36 help='When passed, just print what would happen, but ' | 35 help='When passed, just print what would happen, but ' |
| 37 'do not do it.' | 36 'do not do it.' |
| 38 ) | 37 ) |
| 39 group.add_argument('--event-mon-run-type', default='dry', | 38 group.add_argument('--event-mon-run-type', default='dry', |
| 40 choices=RUNTYPES, | 39 choices=RUNTYPES, |
| 41 help='Determine how to send data. "dry" does not send\n' | 40 help='Determine how to send data. "dry" does not send\n' |
| 42 'anything. "test" sends to the test endpoint, \n' | 41 'anything. "test" sends to the test endpoint, \n' |
| 43 '"prod" to the actual production endpoint, and "file" \n' | 42 '"prod" to the actual production endpoint, and "file" \n' |
| 44 'writes to a file.') | 43 'writes to a file.') |
| 45 group.add_argument('--event-mon-output-file', default='event_mon.output', | 44 group.add_argument('--event-mon-output-file', default='event_mon.output', |
| 46 help='File into which LogEventLite serialized protos are\n' | 45 help='File into which LogEventLite serialized protos are\n' |
| 47 'written when --event-mon-run-type is "file"') | 46 'written when --event-mon-run-type is "file"') |
| 48 group.add_argument('--event-mon-service-name', | 47 group.add_argument('--event-mon-service-name', |
| 49 help='Service name to use in log events.') | 48 help='Service name to use in log events.') |
| 50 group.add_argument('--event-mon-hostname', | 49 group.add_argument('--event-mon-hostname', |
| 51 help='Hostname to use in log events.') | 50 help='Hostname to use in log events.') |
| 52 group.add_argument('--event-mon-appengine-name', | 51 group.add_argument('--event-mon-appengine-name', |
| 53 help='App name to use in log events.') | 52 help='App name to use in log events.') |
| 54 group.add_argument('--event-mon-service-account-creds', | |
| 55 default=DEFAULT_SERVICE_ACCOUNT_CREDS, | |
| 56 metavar='JSON_FILE', | |
| 57 help="Path to a json file containing a service account's" | |
| 58 "\ncredentials. This is relative to the path specified\n" | |
| 59 "in --event-mon-service-accounts-creds-root\n" | |
| 60 "Defaults to '%(default)s'") | |
| 61 group.add_argument('--event-mon-service-accounts-creds-root', | |
| 62 metavar='DIR', | |
| 63 default=infra_libs.SERVICE_ACCOUNTS_CREDS_ROOT, | |
| 64 help="Directory containing service accounts credentials.\n" | |
| 65 "Defaults to %(default)s" | |
| 66 ) | |
| 67 group.add_argument('--event-mon-http-timeout', default=10, type=int, | 53 group.add_argument('--event-mon-http-timeout', default=10, type=int, |
| 68 help='Timeout in seconds for HTTP requests to send events') | 54 help='Timeout in seconds for HTTP requests to send events') |
| 69 group.add_argument('--event-mon-http-retry-backoff', default=2., type=float, | 55 group.add_argument('--event-mon-http-retry-backoff', default=2., type=float, |
| 70 help='Time in seconds before retrying POSTing to the HTTP ' | 56 help='Time in seconds before retrying POSTing to the HTTP ' |
| 71 'endpoint. Randomized exponential backoff is applied on ' | 57 'endpoint. Randomized exponential backoff is applied on ' |
| 72 'subsequent retries.') | 58 'subsequent retries.') |
| 73 | 59 |
| 74 | 60 |
| 75 def process_argparse_options(args): # pragma: no cover | 61 def process_argparse_options(args): # pragma: no cover |
| 76 """Initializes event monitoring based on provided arguments. | 62 """Initializes event monitoring based on provided arguments. |
| 77 | 63 |
| 78 Args: | 64 Args: |
| 79 args(argparse.Namespace): output of ArgumentParser.parse_args. | 65 args(argparse.Namespace): output of ArgumentParser.parse_args. |
| 80 """ | 66 """ |
| 81 setup_monitoring( | 67 setup_monitoring( |
| 82 run_type=args.event_mon_run_type, | 68 run_type=args.event_mon_run_type, |
| 83 hostname=args.event_mon_hostname, | 69 hostname=args.event_mon_hostname, |
| 84 service_name=args.event_mon_service_name, | 70 service_name=args.event_mon_service_name, |
| 85 appengine_name=args.event_mon_appengine_name, | 71 appengine_name=args.event_mon_appengine_name, |
| 86 service_account_creds=args.event_mon_service_account_creds, | |
| 87 service_accounts_creds_root=args.event_mon_service_accounts_creds_root, | |
| 88 output_file=args.event_mon_output_file, | 72 output_file=args.event_mon_output_file, |
| 89 dry_run=args.dry_run, | 73 dry_run=args.dry_run, |
| 90 http_timeout=args.event_mon_http_timeout, | 74 http_timeout=args.event_mon_http_timeout, |
| 91 http_retry_backoff=args.event_mon_http_retry_backoff | 75 http_retry_backoff=args.event_mon_http_retry_backoff |
| 92 ) | 76 ) |
| 93 | 77 |
| 94 | 78 |
| 95 def setup_monitoring(run_type='dry', | 79 def setup_monitoring(run_type='dry', |
| 96 hostname=None, | 80 hostname=None, |
| 97 service_name=None, | 81 service_name=None, |
| 98 appengine_name=None, | 82 appengine_name=None, |
| 99 service_account_creds=None, | |
| 100 service_accounts_creds_root=None, | |
| 101 output_file=None, | 83 output_file=None, |
| 102 dry_run=False, | 84 dry_run=False, |
| 103 http_timeout=10, | 85 http_timeout=10, |
| 104 http_retry_backoff=2.): | 86 http_retry_backoff=2.): |
| 105 """Initializes event monitoring. | 87 """Initializes event monitoring. |
| 106 | 88 |
| 107 This function is mainly used to provide default global values which are | 89 This function is mainly used to provide default global values which are |
| 108 required for the module to work. | 90 required for the module to work. |
| 109 | 91 |
| 110 If you're implementing a command-line tool, use process_argparse_options | 92 If you're implementing a command-line tool, use process_argparse_options |
| 111 instead. | 93 instead. |
| 112 | 94 |
| 113 Args: | 95 Args: |
| 114 run_type (str): One of 'dry', 'test', or 'prod'. Do respectively nothing, | 96 run_type (str): One of 'dry', 'test', or 'prod'. Do respectively nothing, |
| 115 hit the testing endpoint and the production endpoint. | 97 hit the testing endpoint and the production endpoint. |
| 116 | 98 |
| 117 hostname (str): hostname as it should appear in the event. If not provided | 99 hostname (str): hostname as it should appear in the event. If not provided |
| 118 a default value is computed. | 100 a default value is computed. |
| 119 | 101 |
| 120 service_name (str): logical name of the service that emits events. e.g. | 102 service_name (str): logical name of the service that emits events. e.g. |
| 121 "commit_queue". | 103 "commit_queue". |
| 122 | 104 |
| 123 appengine_name (str): name of the appengine app, if running on appengine. | 105 appengine_name (str): name of the appengine app, if running on appengine. |
| 124 | 106 |
| 125 service_account_creds (str): path to a json file containing a service | |
| 126 account's credentials obtained from a Google Cloud project. **Path is | |
| 127 relative to service_account_creds_root**, which is not the current path by | |
| 128 default. See infra_libs.authentication for details. | |
| 129 | |
| 130 service_account_creds_root (str): path containing credentials files. | |
| 131 | |
| 132 output_file (str): file where to write the output in run_type == 'file' | 107 output_file (str): file where to write the output in run_type == 'file' |
| 133 mode. | 108 mode. |
| 134 | 109 |
| 135 dry_run (bool): if True, the code has no side-effect, what would have been | 110 dry_run (bool): if True, the code has no side-effect, what would have been |
| 136 done is printed instead. | 111 done is printed instead. |
| 137 | 112 |
| 138 http_timeout (int): timeout in seconds for HTTP requests to send events. | 113 http_timeout (int): timeout in seconds for HTTP requests to send events. |
| 139 | 114 |
| 140 http_retry_backoff (float): time in seconds before retrying POSTing to the | 115 http_retry_backoff (float): time in seconds before retrying POSTing to the |
| 141 HTTP endpoint. Randomized exponential backoff is applied on subsequent | 116 HTTP endpoint. Randomized exponential backoff is applied on subsequent |
| 142 retries. | 117 retries. |
| 143 """ | 118 """ |
| 144 global _router | 119 global _router |
| 145 logging.debug('event_mon: setting up monitoring.') | 120 logging.debug('event_mon: setting up monitoring.') |
| 146 | 121 |
| 147 if not _router: | 122 if _router: |
| 148 default_event = ChromeInfraEvent() | 123 return |
| 149 | 124 |
| 150 hostname = hostname or socket.getfqdn() | 125 default_event = ChromeInfraEvent() |
| 151 # hostname might be empty string or None on some systems, who knows. | |
| 152 if hostname: # pragma: no branch | |
| 153 default_event.event_source.host_name = hostname | |
| 154 else: # pragma: no cover | |
| 155 logging.warning('event_mon: unable to determine hostname.') | |
| 156 | 126 |
| 157 if service_name: | 127 hostname = hostname or socket.getfqdn() |
| 158 default_event.event_source.service_name = service_name | 128 # hostname might be empty string or None on some systems, who knows. |
| 159 if appengine_name: | 129 if hostname: # pragma: no branch |
| 160 default_event.event_source.appengine_name = appengine_name | 130 default_event.event_source.host_name = hostname |
| 131 else: # pragma: no cover |
| 132 logging.warning('event_mon: unable to determine hostname.') |
| 161 | 133 |
| 162 _cache['default_event'] = default_event | 134 if service_name: |
| 163 if run_type in ('prod', 'test'): | 135 default_event.event_source.service_name = service_name |
| 164 _cache['service_account_creds'] = service_account_creds | 136 if appengine_name: |
| 165 _cache['service_accounts_creds_root'] = service_accounts_creds_root | 137 default_event.event_source.appengine_name = appengine_name |
| 138 |
| 139 _cache['default_event'] = default_event |
| 140 |
| 141 if run_type not in RUNTYPES: |
| 142 logging.error('Unknown run_type (%s). Setting to "dry"', run_type) |
| 143 run_type = 'dry' |
| 144 |
| 145 if run_type == 'dry': |
| 146 # If we are running on AppEngine or devserver, use logging module. |
| 147 server_software = os.environ.get('SERVER_SOFTWARE', '') |
| 148 if (server_software.startswith('Google App Engine') or |
| 149 server_software.startswith('Development')): |
| 150 _router = ev_router._LoggingStreamRouter() |
| 166 else: | 151 else: |
| 167 _cache['service_account_creds'] = None | 152 _router = ev_router._TextStreamRouter() |
| 168 _cache['service_accounts_creds_root'] = None | 153 elif run_type == 'file': |
| 169 | 154 _router = ev_router._LocalFileRouter(output_file, |
| 170 if run_type not in RUNTYPES: | 155 dry_run=dry_run) |
| 171 logging.error('Unknown run_type (%s). Setting to "dry"', run_type) | 156 else: |
| 172 run_type = 'dry' | 157 _router = ev_router._HttpRouter(_cache, |
| 173 | 158 ENDPOINTS.get(run_type), |
| 174 if run_type == 'dry': | 159 dry_run=dry_run, |
| 175 # If we are running on AppEngine or devserver, use logging module. | 160 timeout=http_timeout, |
| 176 server_software = os.environ.get('SERVER_SOFTWARE', '') | 161 retry_backoff=http_retry_backoff) |
| 177 if (server_software.startswith('Google App Engine') or | |
| 178 server_software.startswith('Development')): | |
| 179 _router = ev_router._LoggingStreamRouter() | |
| 180 else: | |
| 181 _router = ev_router._TextStreamRouter() | |
| 182 elif run_type == 'file': | |
| 183 _router = ev_router._LocalFileRouter(output_file, | |
| 184 dry_run=dry_run) | |
| 185 else: | |
| 186 _router = ev_router._HttpRouter(_cache, | |
| 187 ENDPOINTS.get(run_type), | |
| 188 dry_run=dry_run, | |
| 189 timeout=http_timeout, | |
| 190 retry_backoff=http_retry_backoff) | |
| 191 | 162 |
| 192 | 163 |
| 193 def get_default_event(): | 164 def get_default_event(): |
| 194 """Returns a copy of the default event.""" | 165 """Returns a copy of the default event.""" |
| 195 | 166 |
| 196 # We return a copy here to tell people not to modify the event directly. | 167 # We return a copy here to tell people not to modify the event directly. |
| 197 ret = ChromeInfraEvent() | 168 ret = ChromeInfraEvent() |
| 198 ret.CopyFrom(_cache['default_event']) | 169 ret.CopyFrom(_cache['default_event']) |
| 199 return ret | 170 return ret |
| 200 | 171 |
| (...skipping 20 matching lines...) Expand all Loading... |
| 221 | 192 |
| 222 Call this right before exiting the program. | 193 Call this right before exiting the program. |
| 223 | 194 |
| 224 Returns: | 195 Returns: |
| 225 success (bool): False if an error occured | 196 success (bool): False if an error occured |
| 226 """ | 197 """ |
| 227 global _router, _cache | 198 global _router, _cache |
| 228 _router = None | 199 _router = None |
| 229 _cache = {} | 200 _cache = {} |
| 230 return True | 201 return True |
| OLD | NEW |