Index: Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/pipes.py |
diff --git a/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/pipes.py b/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/pipes.py |
new file mode 100644 |
index 0000000000000000000000000000000000000000..3f1f1e4554c86542effb1d9e16949407881d252c |
--- /dev/null |
+++ b/Tools/Scripts/webkitpy/thirdparty/wpt/wpt/tools/wptserve/wptserve/pipes.py |
@@ -0,0 +1,447 @@ |
+from cgi import escape |
+import gzip as gzip_module |
+import re |
+import time |
+import types |
+import uuid |
+from cStringIO import StringIO |
+ |
+ |
+def resolve_content(response): |
+ rv = "".join(item for item in response.iter_content()) |
+ if type(rv) == unicode: |
+ rv = rv.encode(response.encoding) |
+ return rv |
+ |
+ |
+class Pipeline(object): |
+ pipes = {} |
+ |
+ def __init__(self, pipe_string): |
+ self.pipe_functions = self.parse(pipe_string) |
+ |
+ def parse(self, pipe_string): |
+ functions = [] |
+ for item in PipeTokenizer().tokenize(pipe_string): |
+ if not item: |
+ break |
+ if item[0] == "function": |
+ functions.append((self.pipes[item[1]], [])) |
+ elif item[0] == "argument": |
+ functions[-1][1].append(item[1]) |
+ return functions |
+ |
+ def __call__(self, request, response): |
+ for func, args in self.pipe_functions: |
+ response = func(request, response, *args) |
+ return response |
+ |
+ |
+class PipeTokenizer(object): |
+ def __init__(self): |
+ #This whole class can likely be replaced by some regexps |
+ self.state = None |
+ |
+ def tokenize(self, string): |
+ self.string = string |
+ self.state = self.func_name_state |
+ self._index = 0 |
+ while self.state: |
+ yield self.state() |
+ yield None |
+ |
+ def get_char(self): |
+ if self._index >= len(self.string): |
+ return None |
+ rv = self.string[self._index] |
+ self._index += 1 |
+ return rv |
+ |
+ def func_name_state(self): |
+ rv = "" |
+ while True: |
+ char = self.get_char() |
+ if char is None: |
+ self.state = None |
+ if rv: |
+ return ("function", rv) |
+ else: |
+ return None |
+ elif char == "(": |
+ self.state = self.argument_state |
+ return ("function", rv) |
+ elif char == "|": |
+ if rv: |
+ return ("function", rv) |
+ else: |
+ rv += char |
+ |
+ def argument_state(self): |
+ rv = "" |
+ while True: |
+ char = self.get_char() |
+ if char is None: |
+ self.state = None |
+ return ("argument", rv) |
+ elif char == "\\": |
+ rv += self.get_escape() |
+ if rv is None: |
+ #This should perhaps be an error instead |
+ return ("argument", rv) |
+ elif char == ",": |
+ return ("argument", rv) |
+ elif char == ")": |
+ self.state = self.func_name_state |
+ return ("argument", rv) |
+ else: |
+ rv += char |
+ |
+ def get_escape(self): |
+ char = self.get_char() |
+ escapes = {"n": "\n", |
+ "r": "\r", |
+ "t": "\t"} |
+ return escapes.get(char, char) |
+ |
+ |
+class pipe(object): |
+ def __init__(self, *arg_converters): |
+ self.arg_converters = arg_converters |
+ self.max_args = len(self.arg_converters) |
+ self.min_args = 0 |
+ opt_seen = False |
+ for item in self.arg_converters: |
+ if not opt_seen: |
+ if isinstance(item, opt): |
+ opt_seen = True |
+ else: |
+ self.min_args += 1 |
+ else: |
+ if not isinstance(item, opt): |
+ raise ValueError("Non-optional argument cannot follow optional argument") |
+ |
+ def __call__(self, f): |
+ def inner(request, response, *args): |
+ if not (self.min_args <= len(args) <= self.max_args): |
+ raise ValueError("Expected between %d and %d args, got %d" % |
+ (self.min_args, self.max_args, len(args))) |
+ arg_values = tuple(f(x) for f, x in zip(self.arg_converters, args)) |
+ return f(request, response, *arg_values) |
+ Pipeline.pipes[f.__name__] = inner |
+ #We actually want the undecorated function in the main namespace |
+ return f |
+ |
+ |
+class opt(object): |
+ def __init__(self, f): |
+ self.f = f |
+ |
+ def __call__(self, arg): |
+ return self.f(arg) |
+ |
+ |
+def nullable(func): |
+ def inner(arg): |
+ if arg.lower() == "null": |
+ return None |
+ else: |
+ return func(arg) |
+ return inner |
+ |
+ |
+def boolean(arg): |
+ if arg.lower() in ("true", "1"): |
+ return True |
+ elif arg.lower() in ("false", "0"): |
+ return False |
+ raise ValueError |
+ |
+ |
+@pipe(int) |
+def status(request, response, code): |
+ """Alter the status code. |
+ |
+ :param code: Status code to use for the response.""" |
+ response.status = code |
+ return response |
+ |
+ |
+@pipe(str, str, opt(boolean)) |
+def header(request, response, name, value, append=False): |
+ """Set a HTTP header. |
+ |
+ Replaces any existing HTTP header of the same name unless |
+ append is set, in which case the header is appended without |
+ replacement. |
+ |
+ :param name: Name of the header to set. |
+ :param value: Value to use for the header. |
+ :param append: True if existing headers should not be replaced |
+ """ |
+ if not append: |
+ response.headers.set(name, value) |
+ else: |
+ response.headers.append(name, value) |
+ return response |
+ |
+ |
+@pipe(str) |
+def trickle(request, response, delays): |
+ """Send the response in parts, with time delays. |
+ |
+ :param delays: A string of delays and amounts, in bytes, of the |
+ response to send. Each component is separated by |
+ a colon. Amounts in bytes are plain integers, whilst |
+ delays are floats prefixed with a single d e.g. |
+ d1:100:d2 |
+ Would cause a 1 second delay, would then send 100 bytes |
+ of the file, and then cause a 2 second delay, before sending |
+ the remainder of the file. |
+ |
+ If the last token is of the form rN, instead of sending the |
+ remainder of the file, the previous N instructions will be |
+ repeated until the whole file has been sent e.g. |
+ d1:100:d2:r2 |
+ Causes a delay of 1s, then 100 bytes to be sent, then a 2s delay |
+ and then a further 100 bytes followed by a two second delay |
+ until the response has been fully sent. |
+ """ |
+ def parse_delays(): |
+ parts = delays.split(":") |
+ rv = [] |
+ for item in parts: |
+ if item.startswith("d"): |
+ item_type = "delay" |
+ item = item[1:] |
+ value = float(item) |
+ elif item.startswith("r"): |
+ item_type = "repeat" |
+ value = int(item[1:]) |
+ if not value % 2 == 0: |
+ raise ValueError |
+ else: |
+ item_type = "bytes" |
+ value = int(item) |
+ if len(rv) and rv[-1][0] == item_type: |
+ rv[-1][1] += value |
+ else: |
+ rv.append((item_type, value)) |
+ return rv |
+ |
+ delays = parse_delays() |
+ if not delays: |
+ return response |
+ content = resolve_content(response) |
+ modified_content = [] |
+ offset = [0] |
+ |
+ def sleep(seconds): |
+ def inner(): |
+ time.sleep(seconds) |
+ return "" |
+ return inner |
+ |
+ def add_content(delays, repeat=False): |
+ for i, (item_type, value) in enumerate(delays): |
+ if item_type == "bytes": |
+ modified_content.append(content[offset[0]:offset[0] + value]) |
+ offset[0] += value |
+ elif item_type == "delay": |
+ modified_content.append(sleep(value)) |
+ elif item_type == "repeat": |
+ assert i == len(delays) - 1 |
+ while offset[0] < len(content): |
+ add_content(delays[-(value + 1):-1], True) |
+ |
+ if not repeat and offset[0] < len(content): |
+ modified_content.append(content[offset[0]:]) |
+ |
+ add_content(delays) |
+ |
+ response.content = modified_content |
+ return response |
+ |
+ |
+@pipe(nullable(int), opt(nullable(int))) |
+def slice(request, response, start, end=None): |
+ """Send a byte range of the response body |
+ |
+ :param start: The starting offset. Follows python semantics including |
+ negative numbers. |
+ |
+ :param end: The ending offset, again with python semantics and None |
+ (spelled "null" in a query string) to indicate the end of |
+ the file. |
+ """ |
+ content = resolve_content(response) |
+ response.content = content[start:end] |
+ return response |
+ |
+ |
+class ReplacementTokenizer(object): |
+ def ident(scanner, token): |
+ return ("ident", token) |
+ |
+ def index(scanner, token): |
+ token = token[1:-1] |
+ try: |
+ token = int(token) |
+ except ValueError: |
+ token = unicode(token, "utf8") |
+ return ("index", token) |
+ |
+ def var(scanner, token): |
+ token = token[:-1] |
+ return ("var", token) |
+ |
+ def tokenize(self, string): |
+ return self.scanner.scan(string)[0] |
+ |
+ scanner = re.Scanner([(r"\$\w+:", var), |
+ (r"\$?\w+(?:\(\))?", ident), |
+ (r"\[[^\]]*\]", index)]) |
+ |
+ |
+class FirstWrapper(object): |
+ def __init__(self, params): |
+ self.params = params |
+ |
+ def __getitem__(self, key): |
+ try: |
+ return self.params.first(key) |
+ except KeyError: |
+ return "" |
+ |
+ |
+@pipe() |
+def sub(request, response): |
+ """Substitute environment information about the server and request into the script. |
+ |
+ The format is a very limited template language. Substitutions are |
+ enclosed by {{ and }}. There are several avaliable substitutions: |
+ |
+ host |
+ A simple string value and represents the primary host from which the |
+ tests are being run. |
+ domains |
+ A dictionary of available domains indexed by subdomain name. |
+ ports |
+ A dictionary of lists of ports indexed by protocol. |
+ location |
+ A dictionary of parts of the request URL. Valid keys are |
+ 'server, 'scheme', 'host', 'hostname', 'port', 'path' and 'query'. |
+ 'server' is scheme://host:port, 'host' is hostname:port, and query |
+ includes the leading '?', but other delimiters are omitted. |
+ headers |
+ A dictionary of HTTP headers in the request. |
+ GET |
+ A dictionary of query parameters supplied with the request. |
+ uuid() |
+ A pesudo-random UUID suitable for usage with stash |
+ |
+ So for example in a setup running on localhost with a www |
+ subdomain and a http server on ports 80 and 81:: |
+ |
+ {{host}} => localhost |
+ {{domains[www]}} => www.localhost |
+ {{ports[http][1]}} => 81 |
+ |
+ |
+ It is also possible to assign a value to a variable name, which must start with |
+ the $ character, using the ":" syntax e.g. |
+ |
+ {{$id:uuid()} |
+ |
+ Later substitutions in the same file may then refer to the variable |
+ by name e.g. |
+ |
+ {{$id}} |
+ """ |
+ content = resolve_content(response) |
+ |
+ new_content = template(request, content) |
+ |
+ response.content = new_content |
+ return response |
+ |
+def template(request, content): |
+ #TODO: There basically isn't any error handling here |
+ tokenizer = ReplacementTokenizer() |
+ |
+ variables = {} |
+ |
+ def config_replacement(match): |
+ content, = match.groups() |
+ |
+ tokens = tokenizer.tokenize(content) |
+ |
+ if tokens[0][0] == "var": |
+ variable = tokens[0][1] |
+ tokens = tokens[1:] |
+ else: |
+ variable = None |
+ |
+ assert tokens[0][0] == "ident" and all(item[0] == "index" for item in tokens[1:]), tokens |
+ |
+ field = tokens[0][1] |
+ |
+ if field in variables: |
+ value = variables[field] |
+ elif field == "headers": |
+ value = request.headers |
+ elif field == "GET": |
+ value = FirstWrapper(request.GET) |
+ elif field in request.server.config: |
+ value = request.server.config[tokens[0][1]] |
+ elif field == "location": |
+ value = {"server": "%s://%s:%s" % (request.url_parts.scheme, |
+ request.url_parts.hostname, |
+ request.url_parts.port), |
+ "scheme": request.url_parts.scheme, |
+ "host": "%s:%s" % (request.url_parts.hostname, |
+ request.url_parts.port), |
+ "hostname": request.url_parts.hostname, |
+ "port": request.url_parts.port, |
+ "path": request.url_parts.path, |
+ "query": "?%s" % request.url_parts.query} |
+ elif field == "uuid()": |
+ value = str(uuid.uuid4()) |
+ else: |
+ raise Exception("Undefined template variable %s" % field) |
+ |
+ for item in tokens[1:]: |
+ value = value[item[1]] |
+ |
+ assert isinstance(value, (int,) + types.StringTypes), tokens |
+ |
+ if variable is not None: |
+ variables[variable] = value |
+ |
+ #Should possibly support escaping for other contexts e.g. script |
+ #TODO: read the encoding of the response |
+ return escape(unicode(value)).encode("utf-8") |
+ |
+ template_regexp = re.compile(r"{{([^}]*)}}") |
+ new_content, count = template_regexp.subn(config_replacement, content) |
+ |
+ return new_content |
+ |
+@pipe() |
+def gzip(request, response): |
+ """This pipe gzip-encodes response data. |
+ |
+ It sets (or overwrites) these HTTP headers: |
+ Content-Encoding is set to gzip |
+ Content-Length is set to the length of the compressed content |
+ """ |
+ content = resolve_content(response) |
+ response.headers.set("Content-Encoding", "gzip") |
+ |
+ out = StringIO() |
+ with gzip_module.GzipFile(fileobj=out, mode="w") as f: |
+ f.write(content) |
+ response.content = out.getvalue() |
+ |
+ response.headers.set("Content-Length", len(response.content)) |
+ |
+ return response |