Chromium Code Reviews Chromium Code Reviews
Issue 1154373005:
Introduce WPTServe for running W3C Blink Layout tests (Closed)
Base URL: https://chromium.googlesource.com/chromium/blink.git@master| OLD | NEW |
|---|---|
| (Empty) | |
| 1 # -*- coding: utf-8 -*- | |
| 2 import argparse | |
| 3 import json | |
| 4 import os | |
| 5 import signal | |
| 6 import socket | |
| 7 import sys | |
| 8 import threading | |
| 9 import time | |
| 10 import traceback | |
| 11 import urllib2 | |
| 12 import uuid | |
| 13 from collections import defaultdict | |
| 14 from multiprocessing import Process, Event | |
| 15 | |
| 16 from .. import localpaths | |
| 17 | |
| 18 import sslutils | |
| 19 from wptserve import server as wptserve, handlers | |
| 20 from wptserve.logger import set_logger | |
| 21 sys.path.insert(0, os.path.abspath(os.path.join('..', '..', '..', '..', '..', 'w ebkitpy', 'thirdparty'))) | |
|
burnik
2015/06/12 01:40:53
Hm, so this injected line didn't help fix the mod_
| |
| 22 from mod_pywebsocket import standalone as pywebsocket | |
| 23 | |
| 24 repo_root = localpaths.repo_root | |
| 25 | |
| 26 class WorkersHandler(object): | |
| 27 def __init__(self): | |
| 28 self.handler = handlers.handler(self.handle_request) | |
| 29 | |
| 30 def __call__(self, request, response): | |
| 31 return self.handler(request, response) | |
| 32 | |
| 33 def handle_request(self, request, response): | |
| 34 worker_path = request.url_parts.path.replace(".worker", ".worker.js") | |
| 35 return """<!doctype html> | |
| 36 <meta charset=utf-8> | |
| 37 <script src="/resources/testharness.js"></script> | |
| 38 <script src="/resources/testharnessreport.js"></script> | |
| 39 <div id=log></div> | |
| 40 <script> | |
| 41 fetch_tests_from_worker(new Worker("%s")); | |
| 42 </script> | |
| 43 """ % (worker_path,) | |
| 44 | |
| 45 rewrites = [("GET", "/resources/WebIDLParser.js", "/resources/webidl2/lib/webidl 2.js")] | |
| 46 | |
| 47 subdomains = [u"www", | |
| 48 u"www1", | |
| 49 u"www2", | |
| 50 u"天気の良い日", | |
| 51 u"élève"] | |
| 52 | |
| 53 def default_routes(): | |
| 54 return [("GET", "/tools/runner/*", handlers.file_handler), | |
| 55 ("POST", "/tools/runner/update_manifest.py", handlers.python_script_ handler), | |
| 56 ("*", "/_certs/*", handlers.ErrorHandler(404)), | |
| 57 ("*", "/tools/*", handlers.ErrorHandler(404)), | |
| 58 ("*", "{spec}/tools/*", handlers.ErrorHandler(404)), | |
| 59 ("*", "/serve.py", handlers.ErrorHandler(404)), | |
| 60 ("*", "*.py", handlers.python_script_handler), | |
| 61 ("GET", "*.asis", handlers.as_is_handler), | |
| 62 ("GET", "*.worker", WorkersHandler()), | |
| 63 ("GET", "*", handlers.file_handler),] | |
| 64 | |
| 65 def setup_logger(level): | |
| 66 import logging | |
| 67 global logger | |
| 68 logger = logging.getLogger("web-platform-tests") | |
| 69 logging.basicConfig(level=getattr(logging, level.upper())) | |
| 70 set_logger(logger) | |
| 71 | |
| 72 | |
| 73 def open_socket(port): | |
| 74 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| 75 if port != 0: | |
| 76 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | |
| 77 sock.bind(('127.0.0.1', port)) | |
| 78 sock.listen(5) | |
| 79 return sock | |
| 80 | |
| 81 | |
| 82 def get_port(): | |
| 83 free_socket = open_socket(0) | |
| 84 port = free_socket.getsockname()[1] | |
| 85 logger.debug("Going to use port %s" % port) | |
| 86 free_socket.close() | |
| 87 return port | |
| 88 | |
| 89 | |
| 90 class ServerProc(object): | |
| 91 def __init__(self): | |
| 92 self.proc = None | |
| 93 self.daemon = None | |
| 94 self.stop = Event() | |
| 95 | |
| 96 def start(self, init_func, host, port, paths, routes, bind_hostname, externa l_config, | |
| 97 ssl_config, **kwargs): | |
| 98 self.proc = Process(target=self.create_daemon, | |
| 99 args=(init_func, host, port, paths, routes, bind_hos tname, | |
| 100 external_config, ssl_config)) | |
| 101 self.proc.daemon = True | |
| 102 self.proc.start() | |
| 103 | |
| 104 def create_daemon(self, init_func, host, port, paths, routes, bind_hostname, | |
| 105 external_config, ssl_config, **kwargs): | |
| 106 try: | |
| 107 self.daemon = init_func(host, port, paths, routes, bind_hostname, ex ternal_config, | |
| 108 ssl_config, **kwargs) | |
| 109 except socket.error: | |
| 110 print >> sys.stderr, "Socket error on port %s" % port | |
| 111 raise | |
| 112 except: | |
| 113 print >> sys.stderr, traceback.format_exc() | |
| 114 raise | |
| 115 | |
| 116 if self.daemon: | |
| 117 try: | |
| 118 self.daemon.start(block=False) | |
| 119 try: | |
| 120 self.stop.wait() | |
| 121 except KeyboardInterrupt: | |
| 122 pass | |
| 123 except: | |
| 124 print >> sys.stderr, traceback.format_exc() | |
| 125 raise | |
| 126 | |
| 127 def wait(self): | |
| 128 self.stop.set() | |
| 129 self.proc.join() | |
| 130 | |
| 131 def kill(self): | |
| 132 self.stop.set() | |
| 133 self.proc.terminate() | |
| 134 self.proc.join() | |
| 135 | |
| 136 def is_alive(self): | |
| 137 return self.proc.is_alive() | |
| 138 | |
| 139 | |
| 140 def check_subdomains(host, paths, bind_hostname, ssl_config): | |
| 141 port = get_port() | |
| 142 subdomains = get_subdomains(host) | |
| 143 | |
| 144 wrapper = ServerProc() | |
| 145 wrapper.start(start_http_server, host, port, paths, default_routes(), bind_h ostname, | |
| 146 None, ssl_config) | |
| 147 | |
| 148 connected = False | |
| 149 for i in range(10): | |
| 150 try: | |
| 151 urllib2.urlopen("http://%s:%d/" % (host, port)) | |
| 152 connected = True | |
| 153 break | |
| 154 except urllib2.URLError: | |
| 155 time.sleep(1) | |
| 156 | |
| 157 if not connected: | |
| 158 logger.critical("Failed to connect to test server on http://%s:%s You ma y need to edit /etc/hosts or similar" % (host, port)) | |
| 159 sys.exit(1) | |
| 160 | |
| 161 for subdomain, (punycode, host) in subdomains.iteritems(): | |
| 162 domain = "%s.%s" % (punycode, host) | |
| 163 try: | |
| 164 urllib2.urlopen("http://%s:%d/" % (domain, port)) | |
| 165 except Exception as e: | |
| 166 logger.critical("Failed probing domain %s. You may need to edit /etc /hosts or similar." % domain) | |
| 167 sys.exit(1) | |
| 168 | |
| 169 wrapper.wait() | |
| 170 | |
| 171 | |
| 172 def get_subdomains(host): | |
| 173 #This assumes that the tld is ascii-only or already in punycode | |
| 174 return {subdomain: (subdomain.encode("idna"), host) | |
| 175 for subdomain in subdomains} | |
| 176 | |
| 177 | |
| 178 def start_servers(host, ports, paths, routes, bind_hostname, external_config, ss l_config, | |
| 179 **kwargs): | |
| 180 servers = defaultdict(list) | |
| 181 for scheme, ports in ports.iteritems(): | |
| 182 assert len(ports) == {"http":2}.get(scheme, 1) | |
| 183 | |
| 184 for port in ports: | |
| 185 if port is None: | |
| 186 continue | |
| 187 init_func = {"http":start_http_server, | |
| 188 "https":start_https_server, | |
| 189 "ws":start_ws_server, | |
| 190 "wss":start_wss_server}[scheme] | |
| 191 | |
| 192 server_proc = ServerProc() | |
| 193 server_proc.start(init_func, host, port, paths, routes, bind_hostnam e, | |
| 194 external_config, ssl_config, **kwargs) | |
| 195 servers[scheme].append((port, server_proc)) | |
| 196 | |
| 197 return servers | |
| 198 | |
| 199 | |
| 200 def start_http_server(host, port, paths, routes, bind_hostname, external_config, ssl_config, | |
| 201 **kwargs): | |
| 202 return wptserve.WebTestHttpd(host=host, | |
| 203 port=port, | |
| 204 doc_root=paths["doc_root"], | |
| 205 routes=routes, | |
| 206 rewrites=rewrites, | |
| 207 bind_hostname=bind_hostname, | |
| 208 config=external_config, | |
| 209 use_ssl=False, | |
| 210 key_file=None, | |
| 211 certificate=None, | |
| 212 latency=kwargs.get("latency")) | |
| 213 | |
| 214 | |
| 215 def start_https_server(host, port, paths, routes, bind_hostname, external_config , ssl_config, | |
| 216 **kwargs): | |
| 217 return wptserve.WebTestHttpd(host=host, | |
| 218 port=port, | |
| 219 doc_root=paths["doc_root"], | |
| 220 routes=routes, | |
| 221 rewrites=rewrites, | |
| 222 bind_hostname=bind_hostname, | |
| 223 config=external_config, | |
| 224 use_ssl=True, | |
| 225 key_file=ssl_config["key_path"], | |
| 226 certificate=ssl_config["cert_path"], | |
| 227 encrypt_after_connect=ssl_config["encrypt_after _connect"], | |
| 228 latency=kwargs.get("latency")) | |
| 229 | |
| 230 | |
| 231 class WebSocketDaemon(object): | |
| 232 def __init__(self, host, port, doc_root, handlers_root, log_level, bind_host name, | |
| 233 ssl_config): | |
| 234 self.host = host | |
| 235 cmd_args = ["-p", port, | |
| 236 "-d", doc_root, | |
| 237 "-w", handlers_root, | |
| 238 "--log-level", log_level] | |
| 239 | |
| 240 if ssl_config is not None: | |
| 241 # This is usually done through pywebsocket.main, however we're | |
| 242 # working around that to get the server instance and manually | |
| 243 # setup the wss server. | |
| 244 if pywebsocket._import_ssl(): | |
| 245 tls_module = pywebsocket._TLS_BY_STANDARD_MODULE | |
| 246 elif pywebsocket._import_pyopenssl(): | |
| 247 tls_module = pywebsocket._TLS_BY_PYOPENSSL | |
| 248 else: | |
| 249 print "No SSL module available" | |
| 250 sys.exit(1) | |
| 251 | |
| 252 cmd_args += ["--tls", | |
| 253 "--private-key", ssl_config["key_path"], | |
| 254 "--certificate", ssl_config["cert_path"], | |
| 255 "--tls-module", tls_module] | |
| 256 | |
| 257 if (bind_hostname): | |
| 258 cmd_args = ["-H", host] + cmd_args | |
| 259 opts, args = pywebsocket._parse_args_and_config(cmd_args) | |
| 260 opts.cgi_directories = [] | |
| 261 opts.is_executable_method = None | |
| 262 self.server = pywebsocket.WebSocketServer(opts) | |
| 263 ports = [item[0].getsockname()[1] for item in self.server._sockets] | |
| 264 assert all(item == ports[0] for item in ports) | |
| 265 self.port = ports[0] | |
| 266 self.started = False | |
| 267 self.server_thread = None | |
| 268 | |
| 269 def start(self, block=False): | |
| 270 self.started = True | |
| 271 if block: | |
| 272 self.server.serve_forever() | |
| 273 else: | |
| 274 self.server_thread = threading.Thread(target=self.server.serve_forev er) | |
| 275 self.server_thread.setDaemon(True) # don't hang on exit | |
| 276 self.server_thread.start() | |
| 277 | |
| 278 def stop(self): | |
| 279 """ | |
| 280 Stops the server. | |
| 281 | |
| 282 If the server is not running, this method has no effect. | |
| 283 """ | |
| 284 if self.started: | |
| 285 try: | |
| 286 self.server.shutdown() | |
| 287 self.server.server_close() | |
| 288 self.server_thread.join() | |
| 289 self.server_thread = None | |
| 290 except AttributeError: | |
| 291 pass | |
| 292 self.started = False | |
| 293 self.server = None | |
| 294 | |
| 295 | |
| 296 def start_ws_server(host, port, paths, routes, bind_hostname, external_config, s sl_config, | |
| 297 **kwargs): | |
| 298 return WebSocketDaemon(host, | |
| 299 str(port), | |
| 300 repo_root, | |
| 301 paths["ws_doc_root"], | |
| 302 "debug", | |
| 303 bind_hostname, | |
| 304 ssl_config = None) | |
| 305 | |
| 306 | |
| 307 def start_wss_server(host, port, paths, routes, bind_hostname, external_config, ssl_config, | |
| 308 **kwargs): | |
| 309 return WebSocketDaemon(host, | |
| 310 str(port), | |
| 311 repo_root, | |
| 312 paths["ws_doc_root"], | |
| 313 "debug", | |
| 314 bind_hostname, | |
| 315 ssl_config) | |
| 316 | |
| 317 | |
| 318 def get_ports(config, ssl_environment): | |
| 319 rv = defaultdict(list) | |
| 320 for scheme, ports in config["ports"].iteritems(): | |
| 321 for i, port in enumerate(ports): | |
| 322 if scheme in ["wss", "https"] and not ssl_environment.ssl_enabled: | |
| 323 port = None | |
| 324 if port == "auto": | |
| 325 port = get_port() | |
| 326 else: | |
| 327 port = port | |
| 328 rv[scheme].append(port) | |
| 329 return rv | |
| 330 | |
| 331 | |
| 332 | |
| 333 def normalise_config(config, ports): | |
| 334 host = config["external_host"] if config["external_host"] else config["host" ] | |
| 335 domains = get_subdomains(host) | |
| 336 ports_ = {} | |
| 337 for scheme, ports_used in ports.iteritems(): | |
| 338 ports_[scheme] = ports_used | |
| 339 | |
| 340 for key, value in domains.iteritems(): | |
| 341 domains[key] = ".".join(value) | |
| 342 | |
| 343 domains[""] = host | |
| 344 | |
| 345 ports_ = {} | |
| 346 for scheme, ports_used in ports.iteritems(): | |
| 347 ports_[scheme] = ports_used | |
| 348 | |
| 349 return {"host": host, | |
| 350 "domains": domains, | |
| 351 "ports": ports_} | |
| 352 | |
| 353 | |
| 354 def get_ssl_config(config, external_domains, ssl_environment): | |
| 355 key_path, cert_path = ssl_environment.host_cert_path(external_domains) | |
| 356 return {"key_path": key_path, | |
| 357 "cert_path": cert_path, | |
| 358 "encrypt_after_connect": config["ssl"]["encrypt_after_connect"]} | |
| 359 | |
| 360 | |
| 361 def start(config, ssl_environment, routes, **kwargs): | |
| 362 host = config["host"] | |
| 363 domains = get_subdomains(host) | |
| 364 ports = get_ports(config, ssl_environment) | |
| 365 bind_hostname = config["bind_hostname"] | |
| 366 | |
| 367 paths = {"doc_root": config["doc_root"], | |
| 368 "ws_doc_root": config["ws_doc_root"]} | |
| 369 | |
| 370 external_config = normalise_config(config, ports) | |
| 371 | |
| 372 ssl_config = get_ssl_config(config, external_config["domains"].values(), ssl _environment) | |
| 373 | |
| 374 if config["check_subdomains"]: | |
| 375 check_subdomains(host, paths, bind_hostname, ssl_config) | |
| 376 | |
| 377 servers = start_servers(host, ports, paths, routes, bind_hostname, external_ config, | |
| 378 ssl_config, **kwargs) | |
| 379 | |
| 380 return external_config, servers | |
| 381 | |
| 382 | |
| 383 def iter_procs(servers): | |
| 384 for servers in servers.values(): | |
| 385 for port, server in servers: | |
| 386 yield server.proc | |
| 387 | |
| 388 | |
| 389 def value_set(config, key): | |
| 390 return key in config and config[key] is not None | |
| 391 | |
| 392 | |
| 393 def get_value_or_default(config, key, default=None): | |
| 394 return config[key] if value_set(config, key) else default | |
| 395 | |
| 396 | |
| 397 def set_computed_defaults(config): | |
| 398 if not value_set(config, "doc_root"): | |
| 399 config["doc_root"] = repo_root | |
| 400 | |
| 401 if not value_set(config, "ws_doc_root"): | |
| 402 root = get_value_or_default(config, "doc_root", default=repo_root) | |
| 403 config["ws_doc_root"] = os.path.join(root, "websockets", "handlers") | |
| 404 | |
| 405 | |
| 406 def merge_json(base_obj, override_obj): | |
| 407 rv = {} | |
| 408 for key, value in base_obj.iteritems(): | |
| 409 if key not in override_obj: | |
| 410 rv[key] = value | |
| 411 else: | |
| 412 if isinstance(value, dict): | |
| 413 rv[key] = merge_json(value, override_obj[key]) | |
| 414 else: | |
| 415 rv[key] = override_obj[key] | |
| 416 return rv | |
| 417 | |
| 418 | |
| 419 def get_ssl_environment(config): | |
| 420 implementation_type = config["ssl"]["type"] | |
| 421 cls = sslutils.environments[implementation_type] | |
| 422 try: | |
| 423 kwargs = config["ssl"][implementation_type].copy() | |
| 424 except KeyError: | |
| 425 raise ValueError("%s is not a vaid ssl type." % implementation_type) | |
| 426 return cls(logger, **kwargs) | |
| 427 | |
| 428 | |
| 429 def load_config(default_path, override_path=None, **kwargs): | |
| 430 if os.path.exists(default_path): | |
| 431 with open(default_path) as f: | |
| 432 base_obj = json.load(f) | |
| 433 else: | |
| 434 raise ValueError("Config path %s does not exist" % default_path) | |
| 435 | |
| 436 if os.path.exists(override_path): | |
| 437 with open(override_path) as f: | |
| 438 override_obj = json.load(f) | |
| 439 else: | |
| 440 override_obj = {} | |
| 441 rv = merge_json(base_obj, override_obj) | |
| 442 | |
| 443 if kwargs.get("config_path"): | |
| 444 other_path = os.path.abspath(os.path.expanduser(kwargs.get("config_path" ))) | |
| 445 if os.path.exists(other_path): | |
| 446 base_obj = rv | |
| 447 with open(other_path) as f: | |
| 448 override_obj = json.load(f) | |
| 449 rv = merge_json(base_obj, override_obj) | |
| 450 else: | |
| 451 raise ValueError("Config path %s does not exist" % other_path) | |
| 452 | |
| 453 overriding_path_args = [("doc_root", "Document root"), | |
| 454 ("ws_doc_root", "WebSockets document root")] | |
| 455 for key, title in overriding_path_args: | |
| 456 value = kwargs.get(key) | |
| 457 if value is None: | |
| 458 continue | |
| 459 value = os.path.abspath(os.path.expanduser(value)) | |
| 460 if not os.path.exists(value): | |
| 461 raise ValueError("%s path %s does not exist" % (title, value)) | |
| 462 rv[key] = value | |
| 463 | |
| 464 set_computed_defaults(rv) | |
| 465 return rv | |
| 466 | |
| 467 | |
| 468 def get_parser(): | |
| 469 parser = argparse.ArgumentParser() | |
| 470 parser.add_argument("--latency", type=int, | |
| 471 help="Artificial latency to add before sending http resp onses, in ms") | |
| 472 parser.add_argument("--config", action="store", dest="config_path", | |
| 473 help="Path to external config file") | |
| 474 parser.add_argument("--doc_root", action="store", dest="doc_root", | |
| 475 help="Path to document root. Overrides config.") | |
| 476 parser.add_argument("--ws_doc_root", action="store", dest="ws_doc_root", | |
| 477 help="Path to WebSockets document root. Overrides config .") | |
| 478 return parser | |
| 479 | |
| 480 | |
| 481 def main(): | |
| 482 kwargs = vars(get_parser().parse_args()) | |
| 483 config = load_config("config.default.json", | |
| 484 "config.json", | |
| 485 **kwargs) | |
| 486 | |
| 487 setup_logger(config["log_level"]) | |
| 488 | |
| 489 with get_ssl_environment(config) as ssl_env: | |
| 490 config_, servers = start(config, ssl_env, default_routes(), **kwargs) | |
| 491 | |
| 492 try: | |
| 493 while any(item.is_alive() for item in iter_procs(servers)): | |
| 494 for item in iter_procs(servers): | |
| 495 item.join(1) | |
| 496 except KeyboardInterrupt: | |
| 497 logger.info("Shutting down") | |
| OLD | NEW |
