| OLD | NEW |
|---|
| (Empty) | |
| 1 # -*- coding: utf-8 -*- |
| 2 import argparse |
| 3 import json |
| 4 import os |
| 5 import signal |
| 6 import socket |
| 7 import sys |
| 8 import threading |
| 9 import time |
| 10 import traceback |
| 11 import urllib2 |
| 12 import uuid |
| 13 from collections import defaultdict |
| 14 from multiprocessing import Process, Event |
| 15 |
| 16 from .. import localpaths |
| 17 |
| 18 import sslutils |
| 19 from wptserve import server as wptserve, handlers |
| 20 from wptserve.logger import set_logger |
| 21 from mod_pywebsocket import standalone as pywebsocket |
| 22 |
| 23 repo_root = localpaths.repo_root |
| 24 |
| 25 class WorkersHandler(object): |
| 26 def __init__(self): |
| 27 self.handler = handlers.handler(self.handle_request) |
| 28 |
| 29 def __call__(self, request, response): |
| 30 return self.handler(request, response) |
| 31 |
| 32 def handle_request(self, request, response): |
| 33 worker_path = request.url_parts.path.replace(".worker", ".worker.js") |
| 34 return """<!doctype html> |
| 35 <meta charset=utf-8> |
| 36 <script src="/resources/testharness.js"></script> |
| 37 <script src="/resources/testharnessreport.js"></script> |
| 38 <div id=log></div> |
| 39 <script> |
| 40 fetch_tests_from_worker(new Worker("%s")); |
| 41 </script> |
| 42 """ % (worker_path,) |
| 43 |
| 44 rewrites = [("GET", "/resources/WebIDLParser.js", "/resources/webidl2/lib/webidl
2.js")] |
| 45 |
| 46 subdomains = [u"www", |
| 47 u"www1", |
| 48 u"www2", |
| 49 u"天気の良い日", |
| 50 u"élève"] |
| 51 |
| 52 def default_routes(): |
| 53 return [("GET", "/tools/runner/*", handlers.file_handler), |
| 54 ("POST", "/tools/runner/update_manifest.py", handlers.python_script_
handler), |
| 55 ("*", "/_certs/*", handlers.ErrorHandler(404)), |
| 56 ("*", "/tools/*", handlers.ErrorHandler(404)), |
| 57 ("*", "{spec}/tools/*", handlers.ErrorHandler(404)), |
| 58 ("*", "/serve.py", handlers.ErrorHandler(404)), |
| 59 ("*", "*.py", handlers.python_script_handler), |
| 60 ("GET", "*.asis", handlers.as_is_handler), |
| 61 ("GET", "*.worker", WorkersHandler()), |
| 62 ("GET", "*", handlers.file_handler),] |
| 63 |
| 64 def setup_logger(level): |
| 65 import logging |
| 66 global logger |
| 67 logger = logging.getLogger("web-platform-tests") |
| 68 logging.basicConfig(level=getattr(logging, level.upper())) |
| 69 set_logger(logger) |
| 70 |
| 71 |
| 72 def open_socket(port): |
| 73 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| 74 if port != 0: |
| 75 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
| 76 sock.bind(('127.0.0.1', port)) |
| 77 sock.listen(5) |
| 78 return sock |
| 79 |
| 80 |
| 81 def get_port(): |
| 82 free_socket = open_socket(0) |
| 83 port = free_socket.getsockname()[1] |
| 84 logger.debug("Going to use port %s" % port) |
| 85 free_socket.close() |
| 86 return port |
| 87 |
| 88 |
| 89 class ServerProc(object): |
| 90 def __init__(self): |
| 91 self.proc = None |
| 92 self.daemon = None |
| 93 self.stop = Event() |
| 94 |
| 95 def start(self, init_func, host, port, paths, routes, bind_hostname, externa
l_config, |
| 96 ssl_config, **kwargs): |
| 97 self.proc = Process(target=self.create_daemon, |
| 98 args=(init_func, host, port, paths, routes, bind_hos
tname, |
| 99 external_config, ssl_config)) |
| 100 self.proc.daemon = True |
| 101 self.proc.start() |
| 102 |
| 103 def create_daemon(self, init_func, host, port, paths, routes, bind_hostname, |
| 104 external_config, ssl_config, **kwargs): |
| 105 try: |
| 106 self.daemon = init_func(host, port, paths, routes, bind_hostname, ex
ternal_config, |
| 107 ssl_config, **kwargs) |
| 108 except socket.error: |
| 109 print >> sys.stderr, "Socket error on port %s" % port |
| 110 raise |
| 111 except: |
| 112 print >> sys.stderr, traceback.format_exc() |
| 113 raise |
| 114 |
| 115 if self.daemon: |
| 116 try: |
| 117 self.daemon.start(block=False) |
| 118 try: |
| 119 self.stop.wait() |
| 120 except KeyboardInterrupt: |
| 121 pass |
| 122 except: |
| 123 print >> sys.stderr, traceback.format_exc() |
| 124 raise |
| 125 |
| 126 def wait(self): |
| 127 self.stop.set() |
| 128 self.proc.join() |
| 129 |
| 130 def kill(self): |
| 131 self.stop.set() |
| 132 self.proc.terminate() |
| 133 self.proc.join() |
| 134 |
| 135 def is_alive(self): |
| 136 return self.proc.is_alive() |
| 137 |
| 138 |
| 139 def check_subdomains(host, paths, bind_hostname, ssl_config): |
| 140 port = get_port() |
| 141 subdomains = get_subdomains(host) |
| 142 |
| 143 wrapper = ServerProc() |
| 144 wrapper.start(start_http_server, host, port, paths, default_routes(), bind_h
ostname, |
| 145 None, ssl_config) |
| 146 |
| 147 connected = False |
| 148 for i in range(10): |
| 149 try: |
| 150 urllib2.urlopen("http://%s:%d/" % (host, port)) |
| 151 connected = True |
| 152 break |
| 153 except urllib2.URLError: |
| 154 time.sleep(1) |
| 155 |
| 156 if not connected: |
| 157 logger.critical("Failed to connect to test server on http://%s:%s You ma
y need to edit /etc/hosts or similar" % (host, port)) |
| 158 sys.exit(1) |
| 159 |
| 160 for subdomain, (punycode, host) in subdomains.iteritems(): |
| 161 domain = "%s.%s" % (punycode, host) |
| 162 try: |
| 163 urllib2.urlopen("http://%s:%d/" % (domain, port)) |
| 164 except Exception as e: |
| 165 logger.critical("Failed probing domain %s. You may need to edit /etc
/hosts or similar." % domain) |
| 166 sys.exit(1) |
| 167 |
| 168 wrapper.wait() |
| 169 |
| 170 |
| 171 def get_subdomains(host): |
| 172 #This assumes that the tld is ascii-only or already in punycode |
| 173 return {subdomain: (subdomain.encode("idna"), host) |
| 174 for subdomain in subdomains} |
| 175 |
| 176 |
| 177 def start_servers(host, ports, paths, routes, bind_hostname, external_config, ss
l_config, |
| 178 **kwargs): |
| 179 servers = defaultdict(list) |
| 180 for scheme, ports in ports.iteritems(): |
| 181 assert len(ports) == {"http":2}.get(scheme, 1) |
| 182 |
| 183 for port in ports: |
| 184 if port is None: |
| 185 continue |
| 186 init_func = {"http":start_http_server, |
| 187 "https":start_https_server, |
| 188 "ws":start_ws_server, |
| 189 "wss":start_wss_server}[scheme] |
| 190 |
| 191 server_proc = ServerProc() |
| 192 server_proc.start(init_func, host, port, paths, routes, bind_hostnam
e, |
| 193 external_config, ssl_config, **kwargs) |
| 194 servers[scheme].append((port, server_proc)) |
| 195 |
| 196 return servers |
| 197 |
| 198 |
| 199 def start_http_server(host, port, paths, routes, bind_hostname, external_config,
ssl_config, |
| 200 **kwargs): |
| 201 return wptserve.WebTestHttpd(host=host, |
| 202 port=port, |
| 203 doc_root=paths["doc_root"], |
| 204 routes=routes, |
| 205 rewrites=rewrites, |
| 206 bind_hostname=bind_hostname, |
| 207 config=external_config, |
| 208 use_ssl=False, |
| 209 key_file=None, |
| 210 certificate=None, |
| 211 latency=kwargs.get("latency")) |
| 212 |
| 213 |
| 214 def start_https_server(host, port, paths, routes, bind_hostname, external_config
, ssl_config, |
| 215 **kwargs): |
| 216 return wptserve.WebTestHttpd(host=host, |
| 217 port=port, |
| 218 doc_root=paths["doc_root"], |
| 219 routes=routes, |
| 220 rewrites=rewrites, |
| 221 bind_hostname=bind_hostname, |
| 222 config=external_config, |
| 223 use_ssl=True, |
| 224 key_file=ssl_config["key_path"], |
| 225 certificate=ssl_config["cert_path"], |
| 226 encrypt_after_connect=ssl_config["encrypt_after
_connect"], |
| 227 latency=kwargs.get("latency")) |
| 228 |
| 229 |
| 230 class WebSocketDaemon(object): |
| 231 def __init__(self, host, port, doc_root, handlers_root, log_level, bind_host
name, |
| 232 ssl_config): |
| 233 self.host = host |
| 234 cmd_args = ["-p", port, |
| 235 "-d", doc_root, |
| 236 "-w", handlers_root, |
| 237 "--log-level", log_level] |
| 238 |
| 239 if ssl_config is not None: |
| 240 # This is usually done through pywebsocket.main, however we're |
| 241 # working around that to get the server instance and manually |
| 242 # setup the wss server. |
| 243 if pywebsocket._import_ssl(): |
| 244 tls_module = pywebsocket._TLS_BY_STANDARD_MODULE |
| 245 elif pywebsocket._import_pyopenssl(): |
| 246 tls_module = pywebsocket._TLS_BY_PYOPENSSL |
| 247 else: |
| 248 print "No SSL module available" |
| 249 sys.exit(1) |
| 250 |
| 251 cmd_args += ["--tls", |
| 252 "--private-key", ssl_config["key_path"], |
| 253 "--certificate", ssl_config["cert_path"], |
| 254 "--tls-module", tls_module] |
| 255 |
| 256 if (bind_hostname): |
| 257 cmd_args = ["-H", host] + cmd_args |
| 258 opts, args = pywebsocket._parse_args_and_config(cmd_args) |
| 259 opts.cgi_directories = [] |
| 260 opts.is_executable_method = None |
| 261 self.server = pywebsocket.WebSocketServer(opts) |
| 262 ports = [item[0].getsockname()[1] for item in self.server._sockets] |
| 263 assert all(item == ports[0] for item in ports) |
| 264 self.port = ports[0] |
| 265 self.started = False |
| 266 self.server_thread = None |
| 267 |
| 268 def start(self, block=False): |
| 269 self.started = True |
| 270 if block: |
| 271 self.server.serve_forever() |
| 272 else: |
| 273 self.server_thread = threading.Thread(target=self.server.serve_forev
er) |
| 274 self.server_thread.setDaemon(True) # don't hang on exit |
| 275 self.server_thread.start() |
| 276 |
| 277 def stop(self): |
| 278 """ |
| 279 Stops the server. |
| 280 |
| 281 If the server is not running, this method has no effect. |
| 282 """ |
| 283 if self.started: |
| 284 try: |
| 285 self.server.shutdown() |
| 286 self.server.server_close() |
| 287 self.server_thread.join() |
| 288 self.server_thread = None |
| 289 except AttributeError: |
| 290 pass |
| 291 self.started = False |
| 292 self.server = None |
| 293 |
| 294 |
| 295 def start_ws_server(host, port, paths, routes, bind_hostname, external_config, s
sl_config, |
| 296 **kwargs): |
| 297 return WebSocketDaemon(host, |
| 298 str(port), |
| 299 repo_root, |
| 300 paths["ws_doc_root"], |
| 301 "debug", |
| 302 bind_hostname, |
| 303 ssl_config = None) |
| 304 |
| 305 |
| 306 def start_wss_server(host, port, paths, routes, bind_hostname, external_config,
ssl_config, |
| 307 **kwargs): |
| 308 return WebSocketDaemon(host, |
| 309 str(port), |
| 310 repo_root, |
| 311 paths["ws_doc_root"], |
| 312 "debug", |
| 313 bind_hostname, |
| 314 ssl_config) |
| 315 |
| 316 |
| 317 def get_ports(config, ssl_environment): |
| 318 rv = defaultdict(list) |
| 319 for scheme, ports in config["ports"].iteritems(): |
| 320 for i, port in enumerate(ports): |
| 321 if scheme in ["wss", "https"] and not ssl_environment.ssl_enabled: |
| 322 port = None |
| 323 if port == "auto": |
| 324 port = get_port() |
| 325 else: |
| 326 port = port |
| 327 rv[scheme].append(port) |
| 328 return rv |
| 329 |
| 330 |
| 331 |
| 332 def normalise_config(config, ports): |
| 333 host = config["external_host"] if config["external_host"] else config["host"
] |
| 334 domains = get_subdomains(host) |
| 335 ports_ = {} |
| 336 for scheme, ports_used in ports.iteritems(): |
| 337 ports_[scheme] = ports_used |
| 338 |
| 339 for key, value in domains.iteritems(): |
| 340 domains[key] = ".".join(value) |
| 341 |
| 342 domains[""] = host |
| 343 |
| 344 ports_ = {} |
| 345 for scheme, ports_used in ports.iteritems(): |
| 346 ports_[scheme] = ports_used |
| 347 |
| 348 return {"host": host, |
| 349 "domains": domains, |
| 350 "ports": ports_} |
| 351 |
| 352 |
| 353 def get_ssl_config(config, external_domains, ssl_environment): |
| 354 key_path, cert_path = ssl_environment.host_cert_path(external_domains) |
| 355 return {"key_path": key_path, |
| 356 "cert_path": cert_path, |
| 357 "encrypt_after_connect": config["ssl"]["encrypt_after_connect"]} |
| 358 |
| 359 |
| 360 def start(config, ssl_environment, routes, **kwargs): |
| 361 host = config["host"] |
| 362 domains = get_subdomains(host) |
| 363 ports = get_ports(config, ssl_environment) |
| 364 bind_hostname = config["bind_hostname"] |
| 365 |
| 366 paths = {"doc_root": config["doc_root"], |
| 367 "ws_doc_root": config["ws_doc_root"]} |
| 368 |
| 369 external_config = normalise_config(config, ports) |
| 370 |
| 371 ssl_config = get_ssl_config(config, external_config["domains"].values(), ssl
_environment) |
| 372 |
| 373 if config["check_subdomains"]: |
| 374 check_subdomains(host, paths, bind_hostname, ssl_config) |
| 375 |
| 376 servers = start_servers(host, ports, paths, routes, bind_hostname, external_
config, |
| 377 ssl_config, **kwargs) |
| 378 |
| 379 return external_config, servers |
| 380 |
| 381 |
| 382 def iter_procs(servers): |
| 383 for servers in servers.values(): |
| 384 for port, server in servers: |
| 385 yield server.proc |
| 386 |
| 387 |
| 388 def value_set(config, key): |
| 389 return key in config and config[key] is not None |
| 390 |
| 391 |
| 392 def get_value_or_default(config, key, default=None): |
| 393 return config[key] if value_set(config, key) else default |
| 394 |
| 395 |
| 396 def set_computed_defaults(config): |
| 397 if not value_set(config, "doc_root"): |
| 398 config["doc_root"] = repo_root |
| 399 |
| 400 if not value_set(config, "ws_doc_root"): |
| 401 root = get_value_or_default(config, "doc_root", default=repo_root) |
| 402 config["ws_doc_root"] = os.path.join(root, "websockets", "handlers") |
| 403 |
| 404 |
| 405 def merge_json(base_obj, override_obj): |
| 406 rv = {} |
| 407 for key, value in base_obj.iteritems(): |
| 408 if key not in override_obj: |
| 409 rv[key] = value |
| 410 else: |
| 411 if isinstance(value, dict): |
| 412 rv[key] = merge_json(value, override_obj[key]) |
| 413 else: |
| 414 rv[key] = override_obj[key] |
| 415 return rv |
| 416 |
| 417 |
| 418 def get_ssl_environment(config): |
| 419 implementation_type = config["ssl"]["type"] |
| 420 cls = sslutils.environments[implementation_type] |
| 421 try: |
| 422 kwargs = config["ssl"][implementation_type].copy() |
| 423 except KeyError: |
| 424 raise ValueError("%s is not a vaid ssl type." % implementation_type) |
| 425 return cls(logger, **kwargs) |
| 426 |
| 427 |
| 428 def load_config(default_path, override_path=None, **kwargs): |
| 429 if os.path.exists(default_path): |
| 430 with open(default_path) as f: |
| 431 base_obj = json.load(f) |
| 432 else: |
| 433 raise ValueError("Config path %s does not exist" % default_path) |
| 434 |
| 435 if os.path.exists(override_path): |
| 436 with open(override_path) as f: |
| 437 override_obj = json.load(f) |
| 438 else: |
| 439 override_obj = {} |
| 440 rv = merge_json(base_obj, override_obj) |
| 441 |
| 442 if kwargs.get("config_path"): |
| 443 other_path = os.path.abspath(os.path.expanduser(kwargs.get("config_path"
))) |
| 444 if os.path.exists(other_path): |
| 445 base_obj = rv |
| 446 with open(other_path) as f: |
| 447 override_obj = json.load(f) |
| 448 rv = merge_json(base_obj, override_obj) |
| 449 else: |
| 450 raise ValueError("Config path %s does not exist" % other_path) |
| 451 |
| 452 overriding_path_args = [("doc_root", "Document root"), |
| 453 ("ws_doc_root", "WebSockets document root")] |
| 454 for key, title in overriding_path_args: |
| 455 value = kwargs.get(key) |
| 456 if value is None: |
| 457 continue |
| 458 value = os.path.abspath(os.path.expanduser(value)) |
| 459 if not os.path.exists(value): |
| 460 raise ValueError("%s path %s does not exist" % (title, value)) |
| 461 rv[key] = value |
| 462 |
| 463 set_computed_defaults(rv) |
| 464 return rv |
| 465 |
| 466 |
| 467 def get_parser(): |
| 468 parser = argparse.ArgumentParser() |
| 469 parser.add_argument("--latency", type=int, |
| 470 help="Artificial latency to add before sending http resp
onses, in ms") |
| 471 parser.add_argument("--config", action="store", dest="config_path", |
| 472 help="Path to external config file") |
| 473 parser.add_argument("--doc_root", action="store", dest="doc_root", |
| 474 help="Path to document root. Overrides config.") |
| 475 parser.add_argument("--ws_doc_root", action="store", dest="ws_doc_root", |
| 476 help="Path to WebSockets document root. Overrides config
.") |
| 477 return parser |
| 478 |
| 479 |
| 480 def main(): |
| 481 kwargs = vars(get_parser().parse_args()) |
| 482 config = load_config("config.default.json", |
| 483 "config.json", |
| 484 **kwargs) |
| 485 |
| 486 setup_logger(config["log_level"]) |
| 487 |
| 488 with get_ssl_environment(config) as ssl_env: |
| 489 config_, servers = start(config, ssl_env, default_routes(), **kwargs) |
| 490 |
| 491 try: |
| 492 while any(item.is_alive() for item in iter_procs(servers)): |
| 493 for item in iter_procs(servers): |
| 494 item.join(1) |
| 495 except KeyboardInterrupt: |
| 496 logger.info("Shutting down") |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1154373005:
Introduce WPTServe for running W3C Blink Layout tests (Closed)
Base URL: https://chromium.googlesource.com/chromium/blink.git@master