OLD | NEW |
(Empty) | |
| 1 # cython: language_level = 2 |
| 2 # |
| 3 # Code output module |
| 4 # |
| 5 |
| 6 import cython |
| 7 cython.declare(os=object, re=object, operator=object, |
| 8 Naming=object, Options=object, StringEncoding=object, |
| 9 Utils=object, SourceDescriptor=object, StringIOTree=object, |
| 10 DebugFlags=object, basestring=object) |
| 11 |
| 12 import os |
| 13 import re |
| 14 import sys |
| 15 from string import Template |
| 16 import operator |
| 17 import textwrap |
| 18 |
| 19 try: |
| 20 import hashlib |
| 21 except ImportError: |
| 22 import md5 as hashlib |
| 23 |
| 24 import Naming |
| 25 import Options |
| 26 import StringEncoding |
| 27 from Cython import Utils |
| 28 from Scanning import SourceDescriptor |
| 29 from Cython.StringIOTree import StringIOTree |
| 30 import DebugFlags |
| 31 |
| 32 try: |
| 33 from __builtin__ import basestring |
| 34 except ImportError: |
| 35 from builtins import str as basestring |
| 36 |
| 37 KEYWORDS_MUST_BE_BYTES = sys.version_info < (2,7) |
| 38 |
| 39 |
| 40 non_portable_builtins_map = { |
| 41 # builtins that have different names in different Python versions |
| 42 'bytes' : ('PY_MAJOR_VERSION < 3', 'str'), |
| 43 'unicode' : ('PY_MAJOR_VERSION >= 3', 'str'), |
| 44 'basestring' : ('PY_MAJOR_VERSION >= 3', 'str'), |
| 45 'xrange' : ('PY_MAJOR_VERSION >= 3', 'range'), |
| 46 'raw_input' : ('PY_MAJOR_VERSION >= 3', 'input'), |
| 47 'BaseException' : ('PY_VERSION_HEX < 0x02050000', 'Exception'), |
| 48 } |
| 49 |
| 50 basicsize_builtins_map = { |
| 51 # builtins whose type has a different tp_basicsize than sizeof(...) |
| 52 'PyTypeObject' : 'PyHeapTypeObject', |
| 53 } |
| 54 |
| 55 uncachable_builtins = [ |
| 56 # builtin names that cannot be cached because they may or may not |
| 57 # be available at import time |
| 58 'WindowsError', |
| 59 ] |
| 60 |
| 61 modifier_output_mapper = { |
| 62 'inline': 'CYTHON_INLINE' |
| 63 }.get |
| 64 |
| 65 is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match |
| 66 |
| 67 |
| 68 def get_utility_dir(): |
| 69 # make this a function and not global variables: |
| 70 # http://trac.cython.org/cython_trac/ticket/475 |
| 71 Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 72 return os.path.join(Cython_dir, "Utility") |
| 73 |
| 74 |
| 75 class UtilityCodeBase(object): |
| 76 """ |
| 77 Support for loading utility code from a file. |
| 78 |
| 79 Code sections in the file can be specified as follows: |
| 80 |
| 81 ##### MyUtility.proto ##### |
| 82 |
| 83 [proto declarations] |
| 84 |
| 85 ##### MyUtility.init ##### |
| 86 |
| 87 [code run at module initialization] |
| 88 |
| 89 ##### MyUtility ##### |
| 90 #@requires: MyOtherUtility |
| 91 #@substitute: naming |
| 92 |
| 93 [definitions] |
| 94 |
| 95 for prototypes and implementation respectively. For non-python or |
| 96 -cython files backslashes should be used instead. 5 to 30 comment |
| 97 characters may be used on either side. |
| 98 |
| 99 If the @cname decorator is not used and this is a CythonUtilityCode, |
| 100 one should pass in the 'name' keyword argument to be used for name |
| 101 mangling of such entries. |
| 102 """ |
| 103 |
| 104 is_cython_utility = False |
| 105 requires = None |
| 106 _utility_cache = {} |
| 107 |
| 108 @classmethod |
| 109 def _add_utility(cls, utility, type, lines, begin_lineno, tags=None): |
| 110 if utility is None: |
| 111 return |
| 112 |
| 113 code = '\n'.join(lines) |
| 114 if tags and 'substitute' in tags and tags['substitute'] == set(['naming'
]): |
| 115 del tags['substitute'] |
| 116 try: |
| 117 code = Template(code).substitute(vars(Naming)) |
| 118 except (KeyError, ValueError), e: |
| 119 raise RuntimeError("Error parsing templated utility code of type
'%s' at line %d: %s" % ( |
| 120 type, begin_lineno, e)) |
| 121 |
| 122 # remember correct line numbers at least until after templating |
| 123 code = '\n' * begin_lineno + code |
| 124 |
| 125 if type == 'proto': |
| 126 utility[0] = code |
| 127 elif type == 'impl': |
| 128 utility[1] = code |
| 129 else: |
| 130 all_tags = utility[2] |
| 131 if KEYWORDS_MUST_BE_BYTES: |
| 132 type = type.encode('ASCII') |
| 133 all_tags[type] = code |
| 134 |
| 135 if tags: |
| 136 all_tags = utility[2] |
| 137 for name, values in tags.items(): |
| 138 if KEYWORDS_MUST_BE_BYTES: |
| 139 name = name.encode('ASCII') |
| 140 all_tags.setdefault(name, set()).update(values) |
| 141 |
| 142 @classmethod |
| 143 def load_utilities_from_file(cls, path): |
| 144 utilities = cls._utility_cache.get(path) |
| 145 if utilities: |
| 146 return utilities |
| 147 |
| 148 filename = os.path.join(get_utility_dir(), path) |
| 149 _, ext = os.path.splitext(path) |
| 150 if ext in ('.pyx', '.py', '.pxd', '.pxi'): |
| 151 comment = '#' |
| 152 replace_comments = re.compile(r'^\s*#.*').sub |
| 153 else: |
| 154 comment = '/' |
| 155 replace_comments = re.compile(r'^\s*//.*|^\s*/\*[^*]*\*/').sub |
| 156 match_special = re.compile( |
| 157 (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|' |
| 158 r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)' |
| 159 ) % {'C':comment}).match |
| 160 match_type = re.compile('(.+)[.](proto|impl|init|cleanup)$').match |
| 161 |
| 162 f = Utils.open_source_file(filename, encoding='UTF-8') |
| 163 try: |
| 164 all_lines = f.readlines() |
| 165 finally: |
| 166 f.close() |
| 167 |
| 168 utilities = {} |
| 169 lines = [] |
| 170 tags = {} |
| 171 utility = type = None |
| 172 begin_lineno = 0 |
| 173 |
| 174 for lineno, line in enumerate(all_lines): |
| 175 m = match_special(line) |
| 176 if m: |
| 177 if m.group('name'): |
| 178 cls._add_utility(utility, type, lines, begin_lineno, tags) |
| 179 |
| 180 begin_lineno = lineno + 1 |
| 181 del lines[:] |
| 182 tags.clear() |
| 183 |
| 184 name = m.group('name') |
| 185 mtype = match_type(name) |
| 186 if mtype: |
| 187 name, type = mtype.groups() |
| 188 else: |
| 189 type = 'impl' |
| 190 utility = utilities.setdefault(name, [None, None, {}]) |
| 191 else: |
| 192 tags.setdefault(m.group('tag'), set()).add(m.group('value')) |
| 193 lines.append('') # keep line number correct |
| 194 else: |
| 195 lines.append(replace_comments('', line).rstrip()) |
| 196 |
| 197 if utility is None: |
| 198 raise ValueError("Empty utility code file") |
| 199 |
| 200 # Don't forget to add the last utility code |
| 201 cls._add_utility(utility, type, lines, begin_lineno, tags) |
| 202 |
| 203 cls._utility_cache[path] = utilities |
| 204 return utilities |
| 205 |
| 206 @classmethod |
| 207 def load(cls, util_code_name, from_file=None, **kwargs): |
| 208 """ |
| 209 Load utility code from a file specified by from_file (relative to |
| 210 Cython/Utility) and name util_code_name. If from_file is not given, |
| 211 load it from the file util_code_name.*. There should be only one |
| 212 file matched by this pattern. |
| 213 """ |
| 214 if '::' in util_code_name: |
| 215 from_file, util_code_name = util_code_name.rsplit('::', 1) |
| 216 if not from_file: |
| 217 utility_dir = get_utility_dir() |
| 218 prefix = util_code_name + '.' |
| 219 try: |
| 220 listing = os.listdir(utility_dir) |
| 221 except OSError: |
| 222 # XXX the code below assumes as 'zipimport.zipimporter' instance |
| 223 # XXX should be easy to generalize, but too lazy right now to wr
ite it |
| 224 import zipfile |
| 225 global __loader__ |
| 226 loader = __loader__ |
| 227 archive = loader.archive |
| 228 fileobj = zipfile.ZipFile(archive) |
| 229 listing = [ os.path.basename(name) |
| 230 for name in fileobj.namelist() |
| 231 if os.path.join(archive, name).startswith(utility_di
r)] |
| 232 fileobj.close() |
| 233 files = [ os.path.join(utility_dir, filename) |
| 234 for filename in listing |
| 235 if filename.startswith(prefix) ] |
| 236 if not files: |
| 237 raise ValueError("No match found for utility code " + util_code_
name) |
| 238 if len(files) > 1: |
| 239 raise ValueError("More than one filename match found for utility
code " + util_code_name) |
| 240 from_file = files[0] |
| 241 |
| 242 utilities = cls.load_utilities_from_file(from_file) |
| 243 proto, impl, tags = utilities[util_code_name] |
| 244 |
| 245 if tags: |
| 246 orig_kwargs = kwargs.copy() |
| 247 for name, values in tags.items(): |
| 248 if name in kwargs: |
| 249 continue |
| 250 # only pass lists when we have to: most argument expect one valu
e or None |
| 251 if name == 'requires': |
| 252 if orig_kwargs: |
| 253 values = [cls.load(dep, from_file, **orig_kwargs) |
| 254 for dep in sorted(values)] |
| 255 else: |
| 256 # dependencies are rarely unique, so use load_cached() w
hen we can |
| 257 values = [cls.load_cached(dep, from_file) |
| 258 for dep in sorted(values)] |
| 259 elif not values: |
| 260 values = None |
| 261 elif len(values) == 1: |
| 262 values = values[0] |
| 263 kwargs[name] = values |
| 264 |
| 265 if proto is not None: |
| 266 kwargs['proto'] = proto |
| 267 if impl is not None: |
| 268 kwargs['impl'] = impl |
| 269 |
| 270 if 'name' not in kwargs: |
| 271 kwargs['name'] = util_code_name |
| 272 |
| 273 if 'file' not in kwargs and from_file: |
| 274 kwargs['file'] = from_file |
| 275 return cls(**kwargs) |
| 276 |
| 277 @classmethod |
| 278 def load_cached(cls, utility_code_name, from_file=None, __cache={}): |
| 279 """ |
| 280 Calls .load(), but using a per-type cache based on utility name and file
name. |
| 281 """ |
| 282 key = (cls, from_file, utility_code_name) |
| 283 try: |
| 284 return __cache[key] |
| 285 except KeyError: |
| 286 pass |
| 287 code = __cache[key] = cls.load(utility_code_name, from_file) |
| 288 return code |
| 289 |
| 290 @classmethod |
| 291 def load_as_string(cls, util_code_name, from_file=None, **kwargs): |
| 292 """ |
| 293 Load a utility code as a string. Returns (proto, implementation) |
| 294 """ |
| 295 util = cls.load(util_code_name, from_file, **kwargs) |
| 296 proto, impl = util.proto, util.impl |
| 297 return util.format_code(proto), util.format_code(impl) |
| 298 |
| 299 def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').
sub): |
| 300 """ |
| 301 Format a code section for output. |
| 302 """ |
| 303 if code_string: |
| 304 code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n
' |
| 305 return code_string |
| 306 |
| 307 def __str__(self): |
| 308 return "<%s(%s)" % (type(self).__name__, self.name) |
| 309 |
| 310 def get_tree(self): |
| 311 pass |
| 312 |
| 313 |
| 314 class UtilityCode(UtilityCodeBase): |
| 315 """ |
| 316 Stores utility code to add during code generation. |
| 317 |
| 318 See GlobalState.put_utility_code. |
| 319 |
| 320 hashes/equals by instance |
| 321 |
| 322 proto C prototypes |
| 323 impl implemenation code |
| 324 init code to call on module initialization |
| 325 requires utility code dependencies |
| 326 proto_block the place in the resulting file where the prototype should |
| 327 end up |
| 328 name name of the utility code (or None) |
| 329 file filename of the utility code file this utility was loaded |
| 330 from (or None) |
| 331 """ |
| 332 |
| 333 def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=
None, |
| 334 proto_block='utility_code_proto', name=None, file=None): |
| 335 # proto_block: Which code block to dump prototype in. See GlobalState. |
| 336 self.proto = proto |
| 337 self.impl = impl |
| 338 self.init = init |
| 339 self.cleanup = cleanup |
| 340 self.requires = requires |
| 341 self._cache = {} |
| 342 self.specialize_list = [] |
| 343 self.proto_block = proto_block |
| 344 self.name = name |
| 345 self.file = file |
| 346 |
| 347 def __hash__(self): |
| 348 return hash((self.proto, self.impl)) |
| 349 |
| 350 def __eq__(self, other): |
| 351 if self is other: |
| 352 return True |
| 353 if not isinstance(other, type(self)): |
| 354 return False |
| 355 |
| 356 self_proto = getattr(self, 'proto', None) |
| 357 other_proto = getattr(other, 'proto', None) |
| 358 return (self_proto, self.impl) == (other_proto, other.impl) |
| 359 |
| 360 def none_or_sub(self, s, context): |
| 361 """ |
| 362 Format a string in this utility code with context. If None, do nothing. |
| 363 """ |
| 364 if s is None: |
| 365 return None |
| 366 return s % context |
| 367 |
| 368 def specialize(self, pyrex_type=None, **data): |
| 369 # Dicts aren't hashable... |
| 370 if pyrex_type is not None: |
| 371 data['type'] = pyrex_type.declaration_code('') |
| 372 data['type_name'] = pyrex_type.specialization_name() |
| 373 key = tuple(sorted(data.items())) |
| 374 try: |
| 375 return self._cache[key] |
| 376 except KeyError: |
| 377 if self.requires is None: |
| 378 requires = None |
| 379 else: |
| 380 requires = [r.specialize(data) for r in self.requires] |
| 381 |
| 382 s = self._cache[key] = UtilityCode( |
| 383 self.none_or_sub(self.proto, data), |
| 384 self.none_or_sub(self.impl, data), |
| 385 self.none_or_sub(self.init, data), |
| 386 self.none_or_sub(self.cleanup, data), |
| 387 requires, |
| 388 self.proto_block) |
| 389 |
| 390 self.specialize_list.append(s) |
| 391 return s |
| 392 |
| 393 def inject_string_constants(self, impl, output): |
| 394 """Replace 'PYIDENT("xyz")' by a constant Python identifier cname. |
| 395 """ |
| 396 replacements = {} |
| 397 def externalise(matchobj): |
| 398 name = matchobj.group(1) |
| 399 try: |
| 400 cname = replacements[name] |
| 401 except KeyError: |
| 402 cname = replacements[name] = output.get_interned_identifier( |
| 403 StringEncoding.EncodedString(name)).cname |
| 404 return cname |
| 405 |
| 406 impl = re.sub('PYIDENT\("([^"]+)"\)', externalise, impl) |
| 407 return bool(replacements), impl |
| 408 |
| 409 def put_code(self, output): |
| 410 if self.requires: |
| 411 for dependency in self.requires: |
| 412 output.use_utility_code(dependency) |
| 413 if self.proto: |
| 414 output[self.proto_block].put_or_include( |
| 415 self.format_code(self.proto), |
| 416 '%s_proto' % self.name) |
| 417 if self.impl: |
| 418 impl = self.format_code(self.impl) |
| 419 is_specialised, impl = self.inject_string_constants(impl, output) |
| 420 if not is_specialised: |
| 421 # no module specific adaptations => can be reused |
| 422 output['utility_code_def'].put_or_include( |
| 423 impl, '%s_impl' % self.name) |
| 424 else: |
| 425 output['utility_code_def'].put(impl) |
| 426 if self.init: |
| 427 writer = output['init_globals'] |
| 428 writer.putln("/* %s.init */" % self.name) |
| 429 if isinstance(self.init, basestring): |
| 430 writer.put(self.format_code(self.init)) |
| 431 else: |
| 432 self.init(writer, output.module_pos) |
| 433 writer.putln(writer.error_goto_if_PyErr(output.module_pos)) |
| 434 writer.putln() |
| 435 if self.cleanup and Options.generate_cleanup_code: |
| 436 writer = output['cleanup_globals'] |
| 437 if isinstance(self.cleanup, basestring): |
| 438 writer.put_or_include( |
| 439 self.format_code(self.cleanup), |
| 440 '%s_cleanup' % self.name) |
| 441 else: |
| 442 self.cleanup(writer, output.module_pos) |
| 443 |
| 444 |
| 445 def sub_tempita(s, context, file=None, name=None): |
| 446 "Run tempita on string s with given context." |
| 447 if not s: |
| 448 return None |
| 449 |
| 450 if file: |
| 451 context['__name'] = "%s:%s" % (file, name) |
| 452 elif name: |
| 453 context['__name'] = name |
| 454 |
| 455 from Cython.Tempita import sub |
| 456 return sub(s, **context) |
| 457 |
| 458 class TempitaUtilityCode(UtilityCode): |
| 459 def __init__(self, name=None, proto=None, impl=None, init=None, file=None, c
ontext=None, **kwargs): |
| 460 if context is None: |
| 461 context = {} |
| 462 proto = sub_tempita(proto, context, file, name) |
| 463 impl = sub_tempita(impl, context, file, name) |
| 464 init = sub_tempita(init, context, file, name) |
| 465 super(TempitaUtilityCode, self).__init__( |
| 466 proto, impl, init=init, name=name, file=file, **kwargs) |
| 467 |
| 468 def none_or_sub(self, s, context): |
| 469 """ |
| 470 Format a string in this utility code with context. If None, do nothing. |
| 471 """ |
| 472 if s is None: |
| 473 return None |
| 474 return sub_tempita(s, context, self.file, self.name) |
| 475 |
| 476 |
| 477 class LazyUtilityCode(UtilityCodeBase): |
| 478 """ |
| 479 Utility code that calls a callback with the root code writer when |
| 480 available. Useful when you only have 'env' but not 'code'. |
| 481 """ |
| 482 |
| 483 def __init__(self, callback): |
| 484 self.callback = callback |
| 485 |
| 486 def put_code(self, globalstate): |
| 487 utility = self.callback(globalstate.rootwriter) |
| 488 globalstate.use_utility_code(utility) |
| 489 |
| 490 |
| 491 class FunctionState(object): |
| 492 # return_label string function return point label |
| 493 # error_label string error catch point label |
| 494 # continue_label string loop continue point label |
| 495 # break_label string loop break point label |
| 496 # return_from_error_cleanup_label string |
| 497 # label_counter integer counter for naming labels |
| 498 # in_try_finally boolean inside try of try...finally |
| 499 # exc_vars (string * 3) exception variables for reraise, or None |
| 500 # can_trace boolean line tracing is supported in the current
context |
| 501 |
| 502 # Not used for now, perhaps later |
| 503 def __init__(self, owner, names_taken=set()): |
| 504 self.names_taken = names_taken |
| 505 self.owner = owner |
| 506 |
| 507 self.error_label = None |
| 508 self.label_counter = 0 |
| 509 self.labels_used = set() |
| 510 self.return_label = self.new_label() |
| 511 self.new_error_label() |
| 512 self.continue_label = None |
| 513 self.break_label = None |
| 514 self.yield_labels = [] |
| 515 |
| 516 self.in_try_finally = 0 |
| 517 self.exc_vars = None |
| 518 self.can_trace = False |
| 519 |
| 520 self.temps_allocated = [] # of (name, type, manage_ref, static) |
| 521 self.temps_free = {} # (type, manage_ref) -> list of free vars with same
type/managed status |
| 522 self.temps_used_type = {} # name -> (type, manage_ref) |
| 523 self.temp_counter = 0 |
| 524 self.closure_temps = None |
| 525 |
| 526 # This is used to collect temporaries, useful to find out which temps |
| 527 # need to be privatized in parallel sections |
| 528 self.collect_temps_stack = [] |
| 529 |
| 530 # This is used for the error indicator, which needs to be local to the |
| 531 # function. It used to be global, which relies on the GIL being held. |
| 532 # However, exceptions may need to be propagated through 'nogil' |
| 533 # sections, in which case we introduce a race condition. |
| 534 self.should_declare_error_indicator = False |
| 535 self.uses_error_indicator = False |
| 536 |
| 537 # labels |
| 538 |
| 539 def new_label(self, name=None): |
| 540 n = self.label_counter |
| 541 self.label_counter = n + 1 |
| 542 label = "%s%d" % (Naming.label_prefix, n) |
| 543 if name is not None: |
| 544 label += '_' + name |
| 545 return label |
| 546 |
| 547 def new_yield_label(self): |
| 548 label = self.new_label('resume_from_yield') |
| 549 num_and_label = (len(self.yield_labels) + 1, label) |
| 550 self.yield_labels.append(num_and_label) |
| 551 return num_and_label |
| 552 |
| 553 def new_error_label(self): |
| 554 old_err_lbl = self.error_label |
| 555 self.error_label = self.new_label('error') |
| 556 return old_err_lbl |
| 557 |
| 558 def get_loop_labels(self): |
| 559 return ( |
| 560 self.continue_label, |
| 561 self.break_label) |
| 562 |
| 563 def set_loop_labels(self, labels): |
| 564 (self.continue_label, |
| 565 self.break_label) = labels |
| 566 |
| 567 def new_loop_labels(self): |
| 568 old_labels = self.get_loop_labels() |
| 569 self.set_loop_labels( |
| 570 (self.new_label("continue"), |
| 571 self.new_label("break"))) |
| 572 return old_labels |
| 573 |
| 574 def get_all_labels(self): |
| 575 return ( |
| 576 self.continue_label, |
| 577 self.break_label, |
| 578 self.return_label, |
| 579 self.error_label) |
| 580 |
| 581 def set_all_labels(self, labels): |
| 582 (self.continue_label, |
| 583 self.break_label, |
| 584 self.return_label, |
| 585 self.error_label) = labels |
| 586 |
| 587 def all_new_labels(self): |
| 588 old_labels = self.get_all_labels() |
| 589 new_labels = [] |
| 590 for old_label, name in zip(old_labels, ['continue', 'break', 'return', '
error']): |
| 591 if old_label: |
| 592 new_labels.append(self.new_label(name)) |
| 593 else: |
| 594 new_labels.append(old_label) |
| 595 self.set_all_labels(new_labels) |
| 596 return old_labels |
| 597 |
| 598 def use_label(self, lbl): |
| 599 self.labels_used.add(lbl) |
| 600 |
| 601 def label_used(self, lbl): |
| 602 return lbl in self.labels_used |
| 603 |
| 604 # temp handling |
| 605 |
| 606 def allocate_temp(self, type, manage_ref, static=False): |
| 607 """ |
| 608 Allocates a temporary (which may create a new one or get a previously |
| 609 allocated and released one of the same type). Type is simply registered |
| 610 and handed back, but will usually be a PyrexType. |
| 611 |
| 612 If type.is_pyobject, manage_ref comes into play. If manage_ref is set to |
| 613 True, the temp will be decref-ed on return statements and in exception |
| 614 handling clauses. Otherwise the caller has to deal with any reference |
| 615 counting of the variable. |
| 616 |
| 617 If not type.is_pyobject, then manage_ref will be ignored, but it |
| 618 still has to be passed. It is recommended to pass False by convention |
| 619 if it is known that type will never be a Python object. |
| 620 |
| 621 static=True marks the temporary declaration with "static". |
| 622 This is only used when allocating backing store for a module-level |
| 623 C array literals. |
| 624 |
| 625 A C string referring to the variable is returned. |
| 626 """ |
| 627 if type.is_const: |
| 628 type = type.const_base_type |
| 629 if not type.is_pyobject and not type.is_memoryviewslice: |
| 630 # Make manage_ref canonical, so that manage_ref will always mean |
| 631 # a decref is needed. |
| 632 manage_ref = False |
| 633 |
| 634 freelist = self.temps_free.get((type, manage_ref)) |
| 635 if freelist is not None and len(freelist) > 0: |
| 636 result = freelist.pop() |
| 637 else: |
| 638 while True: |
| 639 self.temp_counter += 1 |
| 640 result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_coun
ter) |
| 641 if not result in self.names_taken: break |
| 642 self.temps_allocated.append((result, type, manage_ref, static)) |
| 643 self.temps_used_type[result] = (type, manage_ref) |
| 644 if DebugFlags.debug_temp_code_comments: |
| 645 self.owner.putln("/* %s allocated */" % result) |
| 646 |
| 647 if self.collect_temps_stack: |
| 648 self.collect_temps_stack[-1].add((result, type)) |
| 649 |
| 650 return result |
| 651 |
| 652 def release_temp(self, name): |
| 653 """ |
| 654 Releases a temporary so that it can be reused by other code needing |
| 655 a temp of the same type. |
| 656 """ |
| 657 type, manage_ref = self.temps_used_type[name] |
| 658 freelist = self.temps_free.get((type, manage_ref)) |
| 659 if freelist is None: |
| 660 freelist = [] |
| 661 self.temps_free[(type, manage_ref)] = freelist |
| 662 if name in freelist: |
| 663 raise RuntimeError("Temp %s freed twice!" % name) |
| 664 freelist.append(name) |
| 665 if DebugFlags.debug_temp_code_comments: |
| 666 self.owner.putln("/* %s released */" % name) |
| 667 |
| 668 def temps_in_use(self): |
| 669 """Return a list of (cname,type,manage_ref) tuples of temp names and the
ir type |
| 670 that are currently in use. |
| 671 """ |
| 672 used = [] |
| 673 for name, type, manage_ref, static in self.temps_allocated: |
| 674 freelist = self.temps_free.get((type, manage_ref)) |
| 675 if freelist is None or name not in freelist: |
| 676 used.append((name, type, manage_ref and type.is_pyobject)) |
| 677 return used |
| 678 |
| 679 def temps_holding_reference(self): |
| 680 """Return a list of (cname,type) tuples of temp names and their type |
| 681 that are currently in use. This includes only temps of a |
| 682 Python object type which owns its reference. |
| 683 """ |
| 684 return [(name, type) |
| 685 for name, type, manage_ref in self.temps_in_use() |
| 686 if manage_ref and type.is_pyobject] |
| 687 |
| 688 def all_managed_temps(self): |
| 689 """Return a list of (cname, type) tuples of refcount-managed Python obje
cts. |
| 690 """ |
| 691 return [(cname, type) |
| 692 for cname, type, manage_ref, static in self.temps_allocated |
| 693 if manage_ref] |
| 694 |
| 695 def all_free_managed_temps(self): |
| 696 """Return a list of (cname, type) tuples of refcount-managed Python |
| 697 objects that are not currently in use. This is used by |
| 698 try-except and try-finally blocks to clean up temps in the |
| 699 error case. |
| 700 """ |
| 701 return [(cname, type) |
| 702 for (type, manage_ref), freelist in self.temps_free.items() |
| 703 if manage_ref |
| 704 for cname in freelist] |
| 705 |
| 706 def start_collecting_temps(self): |
| 707 """ |
| 708 Useful to find out which temps were used in a code block |
| 709 """ |
| 710 self.collect_temps_stack.append(set()) |
| 711 |
| 712 def stop_collecting_temps(self): |
| 713 return self.collect_temps_stack.pop() |
| 714 |
| 715 def init_closure_temps(self, scope): |
| 716 self.closure_temps = ClosureTempAllocator(scope) |
| 717 |
| 718 |
| 719 class NumConst(object): |
| 720 """Global info about a Python number constant held by GlobalState. |
| 721 |
| 722 cname string |
| 723 value string |
| 724 py_type string int, long, float |
| 725 value_code string evaluation code if different from value |
| 726 """ |
| 727 |
| 728 def __init__(self, cname, value, py_type, value_code=None): |
| 729 self.cname = cname |
| 730 self.value = value |
| 731 self.py_type = py_type |
| 732 self.value_code = value_code or value |
| 733 |
| 734 |
| 735 class PyObjectConst(object): |
| 736 """Global info about a generic constant held by GlobalState. |
| 737 """ |
| 738 # cname string |
| 739 # type PyrexType |
| 740 |
| 741 def __init__(self, cname, type): |
| 742 self.cname = cname |
| 743 self.type = type |
| 744 |
| 745 |
| 746 cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=obj
ect, |
| 747 replace_identifier=object, find_alphanums=object) |
| 748 possible_unicode_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match |
| 749 possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match |
| 750 replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub |
| 751 find_alphanums = re.compile('([a-zA-Z0-9]+)').findall |
| 752 |
| 753 class StringConst(object): |
| 754 """Global info about a C string constant held by GlobalState. |
| 755 """ |
| 756 # cname string |
| 757 # text EncodedString or BytesLiteral |
| 758 # py_strings {(identifier, encoding) : PyStringConst} |
| 759 |
| 760 def __init__(self, cname, text, byte_string): |
| 761 self.cname = cname |
| 762 self.text = text |
| 763 self.escaped_value = StringEncoding.escape_byte_string(byte_string) |
| 764 self.py_strings = None |
| 765 self.py_versions = [] |
| 766 |
| 767 def add_py_version(self, version): |
| 768 if not version: |
| 769 self.py_versions = [2,3] |
| 770 elif version not in self.py_versions: |
| 771 self.py_versions.append(version) |
| 772 |
| 773 def get_py_string_const(self, encoding, identifier=None, |
| 774 is_str=False, py3str_cstring=None): |
| 775 py_strings = self.py_strings |
| 776 text = self.text |
| 777 |
| 778 is_str = bool(identifier or is_str) |
| 779 is_unicode = encoding is None and not is_str |
| 780 |
| 781 if encoding is None: |
| 782 # unicode string |
| 783 encoding_key = None |
| 784 else: |
| 785 # bytes or str |
| 786 encoding = encoding.lower() |
| 787 if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'): |
| 788 encoding = None |
| 789 encoding_key = None |
| 790 else: |
| 791 encoding_key = ''.join(find_alphanums(encoding)) |
| 792 |
| 793 key = (is_str, is_unicode, encoding_key, py3str_cstring) |
| 794 if py_strings is not None: |
| 795 try: |
| 796 return py_strings[key] |
| 797 except KeyError: |
| 798 pass |
| 799 else: |
| 800 self.py_strings = {} |
| 801 |
| 802 if identifier: |
| 803 intern = True |
| 804 elif identifier is None: |
| 805 if isinstance(text, unicode): |
| 806 intern = bool(possible_unicode_identifier(text)) |
| 807 else: |
| 808 intern = bool(possible_bytes_identifier(text)) |
| 809 else: |
| 810 intern = False |
| 811 if intern: |
| 812 prefix = Naming.interned_prefixes['str'] |
| 813 else: |
| 814 prefix = Naming.py_const_prefix |
| 815 |
| 816 if encoding_key: |
| 817 encoding_prefix = '_%s' % encoding_key |
| 818 else: |
| 819 encoding_prefix = '' |
| 820 |
| 821 pystring_cname = "%s%s%s_%s" % ( |
| 822 prefix, |
| 823 (is_str and 's') or (is_unicode and 'u') or 'b', |
| 824 encoding_prefix, |
| 825 self.cname[len(Naming.const_prefix):]) |
| 826 |
| 827 py_string = PyStringConst( |
| 828 pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern
) |
| 829 self.py_strings[key] = py_string |
| 830 return py_string |
| 831 |
| 832 class PyStringConst(object): |
| 833 """Global info about a Python string constant held by GlobalState. |
| 834 """ |
| 835 # cname string |
| 836 # py3str_cstring string |
| 837 # encoding string |
| 838 # intern boolean |
| 839 # is_unicode boolean |
| 840 # is_str boolean |
| 841 |
| 842 def __init__(self, cname, encoding, is_unicode, is_str=False, |
| 843 py3str_cstring=None, intern=False): |
| 844 self.cname = cname |
| 845 self.py3str_cstring = py3str_cstring |
| 846 self.encoding = encoding |
| 847 self.is_str = is_str |
| 848 self.is_unicode = is_unicode |
| 849 self.intern = intern |
| 850 |
| 851 def __lt__(self, other): |
| 852 return self.cname < other.cname |
| 853 |
| 854 |
| 855 class GlobalState(object): |
| 856 # filename_table {string : int} for finding filename table indexes |
| 857 # filename_list [string] filenames in filename table order |
| 858 # input_file_contents dict contents (=list of lines) of any file tha
t was used as input |
| 859 # to create this output C code. This is |
| 860 # used to annotate the comments. |
| 861 # |
| 862 # utility_codes set IDs of used utility code (to avoid rein
sertion) |
| 863 # |
| 864 # declared_cnames {string:Entry} used in a transition phase to merge pxd-d
eclared |
| 865 # constants etc. into the pyx-declared ones
(i.e, |
| 866 # check if constants are already added). |
| 867 # In time, hopefully the literals etc. will
be |
| 868 # supplied directly instead. |
| 869 # |
| 870 # const_cnames_used dict global counter for unique constant identi
fiers |
| 871 # |
| 872 |
| 873 # parts {string:CCodeWriter} |
| 874 |
| 875 |
| 876 # interned_strings |
| 877 # consts |
| 878 # interned_nums |
| 879 |
| 880 # directives set Temporary variable used to track |
| 881 # the current set of directives in the code
generation |
| 882 # process. |
| 883 |
| 884 directives = {} |
| 885 |
| 886 code_layout = [ |
| 887 'h_code', |
| 888 'filename_table', |
| 889 'utility_code_proto_before_types', |
| 890 'numeric_typedefs', # Let these detailed individual parts stay!
, |
| 891 'complex_type_declarations', # as the proper solution is to make a full
DAG... |
| 892 'type_declarations', # More coarse-grained blocks would simply h
ide |
| 893 'utility_code_proto', # the ugliness, not fix it |
| 894 'module_declarations', |
| 895 'typeinfo', |
| 896 'before_global_var', |
| 897 'global_var', |
| 898 'decls', |
| 899 'all_the_rest', |
| 900 'pystring_table', |
| 901 'cached_builtins', |
| 902 'cached_constants', |
| 903 'init_globals', |
| 904 'init_module', |
| 905 'cleanup_globals', |
| 906 'cleanup_module', |
| 907 'main_method', |
| 908 'utility_code_def', |
| 909 'end' |
| 910 ] |
| 911 |
| 912 |
| 913 def __init__(self, writer, module_node, emit_linenums=False, common_utility_
include_dir=None): |
| 914 self.filename_table = {} |
| 915 self.filename_list = [] |
| 916 self.input_file_contents = {} |
| 917 self.utility_codes = set() |
| 918 self.declared_cnames = {} |
| 919 self.in_utility_code_generation = False |
| 920 self.emit_linenums = emit_linenums |
| 921 self.common_utility_include_dir = common_utility_include_dir |
| 922 self.parts = {} |
| 923 self.module_node = module_node # because some utility code generation ne
eds it |
| 924 # (generating backwards-compatible Get/Re
leaseBuffer |
| 925 |
| 926 self.const_cnames_used = {} |
| 927 self.string_const_index = {} |
| 928 self.pyunicode_ptr_const_index = {} |
| 929 self.num_const_index = {} |
| 930 self.py_constants = [] |
| 931 |
| 932 assert writer.globalstate is None |
| 933 writer.globalstate = self |
| 934 self.rootwriter = writer |
| 935 |
| 936 def initialize_main_c_code(self): |
| 937 rootwriter = self.rootwriter |
| 938 for part in self.code_layout: |
| 939 self.parts[part] = rootwriter.insertion_point() |
| 940 |
| 941 if not Options.cache_builtins: |
| 942 del self.parts['cached_builtins'] |
| 943 else: |
| 944 w = self.parts['cached_builtins'] |
| 945 w.enter_cfunc_scope() |
| 946 w.putln("static int __Pyx_InitCachedBuiltins(void) {") |
| 947 |
| 948 w = self.parts['cached_constants'] |
| 949 w.enter_cfunc_scope() |
| 950 w.putln("") |
| 951 w.putln("static int __Pyx_InitCachedConstants(void) {") |
| 952 w.put_declare_refcount_context() |
| 953 w.put_setup_refcount_context("__Pyx_InitCachedConstants") |
| 954 |
| 955 w = self.parts['init_globals'] |
| 956 w.enter_cfunc_scope() |
| 957 w.putln("") |
| 958 w.putln("static int __Pyx_InitGlobals(void) {") |
| 959 |
| 960 if not Options.generate_cleanup_code: |
| 961 del self.parts['cleanup_globals'] |
| 962 else: |
| 963 w = self.parts['cleanup_globals'] |
| 964 w.enter_cfunc_scope() |
| 965 w.putln("") |
| 966 w.putln("static void __Pyx_CleanupGlobals(void) {") |
| 967 |
| 968 # |
| 969 # utility_code_def |
| 970 # |
| 971 code = self.parts['utility_code_def'] |
| 972 if self.emit_linenums: |
| 973 code.write('\n#line 1 "cython_utility"\n') |
| 974 code.putln("") |
| 975 code.putln("/* Runtime support code */") |
| 976 |
| 977 def finalize_main_c_code(self): |
| 978 self.close_global_decls() |
| 979 |
| 980 # |
| 981 # utility_code_def |
| 982 # |
| 983 code = self.parts['utility_code_def'] |
| 984 code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c
")[1]) |
| 985 code.putln("") |
| 986 |
| 987 def __getitem__(self, key): |
| 988 return self.parts[key] |
| 989 |
| 990 # |
| 991 # Global constants, interned objects, etc. |
| 992 # |
| 993 def close_global_decls(self): |
| 994 # This is called when it is known that no more global declarations will |
| 995 # declared. |
| 996 self.generate_const_declarations() |
| 997 if Options.cache_builtins: |
| 998 w = self.parts['cached_builtins'] |
| 999 w.putln("return 0;") |
| 1000 if w.label_used(w.error_label): |
| 1001 w.put_label(w.error_label) |
| 1002 w.putln("return -1;") |
| 1003 w.putln("}") |
| 1004 w.exit_cfunc_scope() |
| 1005 |
| 1006 w = self.parts['cached_constants'] |
| 1007 w.put_finish_refcount_context() |
| 1008 w.putln("return 0;") |
| 1009 if w.label_used(w.error_label): |
| 1010 w.put_label(w.error_label) |
| 1011 w.put_finish_refcount_context() |
| 1012 w.putln("return -1;") |
| 1013 w.putln("}") |
| 1014 w.exit_cfunc_scope() |
| 1015 |
| 1016 w = self.parts['init_globals'] |
| 1017 w.putln("return 0;") |
| 1018 if w.label_used(w.error_label): |
| 1019 w.put_label(w.error_label) |
| 1020 w.putln("return -1;") |
| 1021 w.putln("}") |
| 1022 w.exit_cfunc_scope() |
| 1023 |
| 1024 if Options.generate_cleanup_code: |
| 1025 w = self.parts['cleanup_globals'] |
| 1026 w.putln("}") |
| 1027 w.exit_cfunc_scope() |
| 1028 |
| 1029 if Options.generate_cleanup_code: |
| 1030 w = self.parts['cleanup_module'] |
| 1031 w.putln("}") |
| 1032 w.exit_cfunc_scope() |
| 1033 |
| 1034 def put_pyobject_decl(self, entry): |
| 1035 self['global_var'].putln("static PyObject *%s;" % entry.cname) |
| 1036 |
| 1037 # constant handling at code generation time |
| 1038 |
| 1039 def get_cached_constants_writer(self): |
| 1040 return self.parts['cached_constants'] |
| 1041 |
| 1042 def get_int_const(self, str_value, longness=False): |
| 1043 py_type = longness and 'long' or 'int' |
| 1044 try: |
| 1045 c = self.num_const_index[(str_value, py_type)] |
| 1046 except KeyError: |
| 1047 c = self.new_num_const(str_value, py_type) |
| 1048 return c |
| 1049 |
| 1050 def get_float_const(self, str_value, value_code): |
| 1051 try: |
| 1052 c = self.num_const_index[(str_value, 'float')] |
| 1053 except KeyError: |
| 1054 c = self.new_num_const(str_value, 'float', value_code) |
| 1055 return c |
| 1056 |
| 1057 def get_py_const(self, type, prefix='', cleanup_level=None): |
| 1058 # create a new Python object constant |
| 1059 const = self.new_py_const(type, prefix) |
| 1060 if cleanup_level is not None \ |
| 1061 and cleanup_level <= Options.generate_cleanup_code: |
| 1062 cleanup_writer = self.parts['cleanup_globals'] |
| 1063 cleanup_writer.putln('Py_CLEAR(%s);' % const.cname) |
| 1064 return const |
| 1065 |
| 1066 def get_string_const(self, text, py_version=None): |
| 1067 # return a C string constant, creating a new one if necessary |
| 1068 if text.is_unicode: |
| 1069 byte_string = text.utf8encode() |
| 1070 else: |
| 1071 byte_string = text.byteencode() |
| 1072 try: |
| 1073 c = self.string_const_index[byte_string] |
| 1074 except KeyError: |
| 1075 c = self.new_string_const(text, byte_string) |
| 1076 c.add_py_version(py_version) |
| 1077 return c |
| 1078 |
| 1079 def get_pyunicode_ptr_const(self, text): |
| 1080 # return a Py_UNICODE[] constant, creating a new one if necessary |
| 1081 assert text.is_unicode |
| 1082 try: |
| 1083 c = self.pyunicode_ptr_const_index[text] |
| 1084 except KeyError: |
| 1085 c = self.pyunicode_ptr_const_index[text] = self.new_const_cname() |
| 1086 return c |
| 1087 |
| 1088 def get_py_string_const(self, text, identifier=None, |
| 1089 is_str=False, unicode_value=None): |
| 1090 # return a Python string constant, creating a new one if necessary |
| 1091 py3str_cstring = None |
| 1092 if is_str and unicode_value is not None \ |
| 1093 and unicode_value.utf8encode() != text.byteencode(): |
| 1094 py3str_cstring = self.get_string_const(unicode_value, py_version=3) |
| 1095 c_string = self.get_string_const(text, py_version=2) |
| 1096 else: |
| 1097 c_string = self.get_string_const(text) |
| 1098 py_string = c_string.get_py_string_const( |
| 1099 text.encoding, identifier, is_str, py3str_cstring) |
| 1100 return py_string |
| 1101 |
| 1102 def get_interned_identifier(self, text): |
| 1103 return self.get_py_string_const(text, identifier=True) |
| 1104 |
| 1105 def new_string_const(self, text, byte_string): |
| 1106 cname = self.new_string_const_cname(byte_string) |
| 1107 c = StringConst(cname, text, byte_string) |
| 1108 self.string_const_index[byte_string] = c |
| 1109 return c |
| 1110 |
| 1111 def new_num_const(self, value, py_type, value_code=None): |
| 1112 cname = self.new_num_const_cname(value, py_type) |
| 1113 c = NumConst(cname, value, py_type, value_code) |
| 1114 self.num_const_index[(value, py_type)] = c |
| 1115 return c |
| 1116 |
| 1117 def new_py_const(self, type, prefix=''): |
| 1118 cname = self.new_const_cname(prefix) |
| 1119 c = PyObjectConst(cname, type) |
| 1120 self.py_constants.append(c) |
| 1121 return c |
| 1122 |
| 1123 def new_string_const_cname(self, bytes_value): |
| 1124 # Create a new globally-unique nice name for a C string constant. |
| 1125 value = bytes_value.decode('ASCII', 'ignore') |
| 1126 return self.new_const_cname(value=value) |
| 1127 |
| 1128 def new_num_const_cname(self, value, py_type): |
| 1129 if py_type == 'long': |
| 1130 value += 'L' |
| 1131 py_type = 'int' |
| 1132 prefix = Naming.interned_prefixes[py_type] |
| 1133 cname = "%s%s" % (prefix, value) |
| 1134 cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_') |
| 1135 return cname |
| 1136 |
| 1137 def new_const_cname(self, prefix='', value=''): |
| 1138 value = replace_identifier('_', value)[:32].strip('_') |
| 1139 used = self.const_cnames_used |
| 1140 name_suffix = value |
| 1141 while name_suffix in used: |
| 1142 counter = used[value] = used[value] + 1 |
| 1143 name_suffix = '%s_%d' % (value, counter) |
| 1144 used[name_suffix] = 1 |
| 1145 if prefix: |
| 1146 prefix = Naming.interned_prefixes[prefix] |
| 1147 else: |
| 1148 prefix = Naming.const_prefix |
| 1149 return "%s%s" % (prefix, name_suffix) |
| 1150 |
| 1151 def add_cached_builtin_decl(self, entry): |
| 1152 if entry.is_builtin and entry.is_const: |
| 1153 if self.should_declare(entry.cname, entry): |
| 1154 self.put_pyobject_decl(entry) |
| 1155 w = self.parts['cached_builtins'] |
| 1156 condition = None |
| 1157 if entry.name in non_portable_builtins_map: |
| 1158 condition, replacement = non_portable_builtins_map[entry.nam
e] |
| 1159 w.putln('#if %s' % condition) |
| 1160 self.put_cached_builtin_init( |
| 1161 entry.pos, StringEncoding.EncodedString(replacement), |
| 1162 entry.cname) |
| 1163 w.putln('#else') |
| 1164 self.put_cached_builtin_init( |
| 1165 entry.pos, StringEncoding.EncodedString(entry.name), |
| 1166 entry.cname) |
| 1167 if condition: |
| 1168 w.putln('#endif') |
| 1169 |
| 1170 def put_cached_builtin_init(self, pos, name, cname): |
| 1171 w = self.parts['cached_builtins'] |
| 1172 interned_cname = self.get_interned_identifier(name).cname |
| 1173 self.use_utility_code( |
| 1174 UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c")) |
| 1175 w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % ( |
| 1176 cname, |
| 1177 interned_cname, |
| 1178 cname, |
| 1179 w.error_goto(pos))) |
| 1180 |
| 1181 def generate_const_declarations(self): |
| 1182 self.generate_string_constants() |
| 1183 self.generate_num_constants() |
| 1184 self.generate_object_constant_decls() |
| 1185 |
| 1186 def generate_object_constant_decls(self): |
| 1187 consts = [ (len(c.cname), c.cname, c) |
| 1188 for c in self.py_constants ] |
| 1189 consts.sort() |
| 1190 decls_writer = self.parts['decls'] |
| 1191 for _, cname, c in consts: |
| 1192 decls_writer.putln( |
| 1193 "static %s;" % c.type.declaration_code(cname)) |
| 1194 |
| 1195 def generate_string_constants(self): |
| 1196 c_consts = [ (len(c.cname), c.cname, c) |
| 1197 for c in self.string_const_index.values() ] |
| 1198 c_consts.sort() |
| 1199 py_strings = [] |
| 1200 |
| 1201 decls_writer = self.parts['decls'] |
| 1202 for _, cname, c in c_consts: |
| 1203 conditional = False |
| 1204 if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versio
ns): |
| 1205 conditional = True |
| 1206 decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % ( |
| 1207 (2 in c.py_versions) and '<' or '>=')) |
| 1208 decls_writer.putln('static char %s[] = "%s";' % ( |
| 1209 cname, StringEncoding.split_string_literal(c.escaped_value))) |
| 1210 if conditional: |
| 1211 decls_writer.putln("#endif") |
| 1212 if c.py_strings is not None: |
| 1213 for py_string in c.py_strings.values(): |
| 1214 py_strings.append((c.cname, len(py_string.cname), py_string)
) |
| 1215 |
| 1216 for c, cname in self.pyunicode_ptr_const_index.items(): |
| 1217 utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c) |
| 1218 if utf16_array: |
| 1219 # Narrow and wide representations differ |
| 1220 decls_writer.putln("#ifdef Py_UNICODE_WIDE") |
| 1221 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf3
2_array)) |
| 1222 if utf16_array: |
| 1223 decls_writer.putln("#else") |
| 1224 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname,
utf16_array)) |
| 1225 decls_writer.putln("#endif") |
| 1226 |
| 1227 if py_strings: |
| 1228 self.use_utility_code(UtilityCode.load_cached("InitStrings", "String
Tools.c")) |
| 1229 py_strings.sort() |
| 1230 w = self.parts['pystring_table'] |
| 1231 w.putln("") |
| 1232 w.putln("static __Pyx_StringTabEntry %s[] = {" % |
| 1233 Naming.stringtab_cname) |
| 1234 for c_cname, _, py_string in py_strings: |
| 1235 if not py_string.is_str or not py_string.encoding or \ |
| 1236 py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', |
| 1237 'UTF8', 'UTF-8'): |
| 1238 encoding = '0' |
| 1239 else: |
| 1240 encoding = '"%s"' % py_string.encoding.lower() |
| 1241 |
| 1242 decls_writer.putln( |
| 1243 "static PyObject *%s;" % py_string.cname) |
| 1244 if py_string.py3str_cstring: |
| 1245 w.putln("#if PY_MAJOR_VERSION >= 3") |
| 1246 w.putln( |
| 1247 "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( |
| 1248 py_string.cname, |
| 1249 py_string.py3str_cstring.cname, |
| 1250 py_string.py3str_cstring.cname, |
| 1251 '0', 1, 0, |
| 1252 py_string.intern |
| 1253 )) |
| 1254 w.putln("#else") |
| 1255 w.putln( |
| 1256 "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( |
| 1257 py_string.cname, |
| 1258 c_cname, |
| 1259 c_cname, |
| 1260 encoding, |
| 1261 py_string.is_unicode, |
| 1262 py_string.is_str, |
| 1263 py_string.intern |
| 1264 )) |
| 1265 if py_string.py3str_cstring: |
| 1266 w.putln("#endif") |
| 1267 w.putln("{0, 0, 0, 0, 0, 0, 0}") |
| 1268 w.putln("};") |
| 1269 |
| 1270 init_globals = self.parts['init_globals'] |
| 1271 init_globals.putln( |
| 1272 "if (__Pyx_InitStrings(%s) < 0) %s;" % ( |
| 1273 Naming.stringtab_cname, |
| 1274 init_globals.error_goto(self.module_pos))) |
| 1275 |
| 1276 def generate_num_constants(self): |
| 1277 consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_
code, c) |
| 1278 for c in self.num_const_index.values()] |
| 1279 consts.sort() |
| 1280 decls_writer = self.parts['decls'] |
| 1281 init_globals = self.parts['init_globals'] |
| 1282 for py_type, _, _, value, value_code, c in consts: |
| 1283 cname = c.cname |
| 1284 decls_writer.putln("static PyObject *%s;" % cname) |
| 1285 if py_type == 'float': |
| 1286 function = 'PyFloat_FromDouble(%s)' |
| 1287 elif py_type == 'long': |
| 1288 function = 'PyLong_FromString((char *)"%s", 0, 0)' |
| 1289 elif Utils.long_literal(value): |
| 1290 function = 'PyInt_FromString((char *)"%s", 0, 0)' |
| 1291 elif len(value.lstrip('-')) > 4: |
| 1292 function = "PyInt_FromLong(%sL)" |
| 1293 else: |
| 1294 function = "PyInt_FromLong(%s)" |
| 1295 init_globals.putln('%s = %s; %s' % ( |
| 1296 cname, function % value_code, |
| 1297 init_globals.error_goto_if_null(cname, self.module_pos))) |
| 1298 |
| 1299 # The functions below are there in a transition phase only |
| 1300 # and will be deprecated. They are called from Nodes.BlockNode. |
| 1301 # The copy&paste duplication is intentional in order to be able |
| 1302 # to see quickly how BlockNode worked, until this is replaced. |
| 1303 |
| 1304 def should_declare(self, cname, entry): |
| 1305 if cname in self.declared_cnames: |
| 1306 other = self.declared_cnames[cname] |
| 1307 assert str(entry.type) == str(other.type) |
| 1308 assert entry.init == other.init |
| 1309 return False |
| 1310 else: |
| 1311 self.declared_cnames[cname] = entry |
| 1312 return True |
| 1313 |
| 1314 # |
| 1315 # File name state |
| 1316 # |
| 1317 |
| 1318 def lookup_filename(self, filename): |
| 1319 try: |
| 1320 index = self.filename_table[filename] |
| 1321 except KeyError: |
| 1322 index = len(self.filename_list) |
| 1323 self.filename_list.append(filename) |
| 1324 self.filename_table[filename] = index |
| 1325 return index |
| 1326 |
| 1327 def commented_file_contents(self, source_desc): |
| 1328 try: |
| 1329 return self.input_file_contents[source_desc] |
| 1330 except KeyError: |
| 1331 pass |
| 1332 source_file = source_desc.get_lines(encoding='ASCII', |
| 1333 error_handling='ignore') |
| 1334 try: |
| 1335 F = [u' * ' + line.rstrip().replace( |
| 1336 u'*/', u'*[inserted by cython to avoid comment closer]/' |
| 1337 ).replace( |
| 1338 u'/*', u'/[inserted by cython to avoid comment start]*' |
| 1339 ) |
| 1340 for line in source_file] |
| 1341 finally: |
| 1342 if hasattr(source_file, 'close'): |
| 1343 source_file.close() |
| 1344 if not F: F.append(u'') |
| 1345 self.input_file_contents[source_desc] = F |
| 1346 return F |
| 1347 |
| 1348 # |
| 1349 # Utility code state |
| 1350 # |
| 1351 |
| 1352 def use_utility_code(self, utility_code): |
| 1353 """ |
| 1354 Adds code to the C file. utility_code should |
| 1355 a) implement __eq__/__hash__ for the purpose of knowing whether the same |
| 1356 code has already been included |
| 1357 b) implement put_code, which takes a globalstate instance |
| 1358 |
| 1359 See UtilityCode. |
| 1360 """ |
| 1361 if utility_code not in self.utility_codes: |
| 1362 self.utility_codes.add(utility_code) |
| 1363 utility_code.put_code(self) |
| 1364 |
| 1365 |
| 1366 def funccontext_property(name): |
| 1367 attribute_of = operator.attrgetter(name) |
| 1368 def get(self): |
| 1369 return attribute_of(self.funcstate) |
| 1370 def set(self, value): |
| 1371 setattr(self.funcstate, name, value) |
| 1372 return property(get, set) |
| 1373 |
| 1374 |
| 1375 class CCodeWriter(object): |
| 1376 """ |
| 1377 Utility class to output C code. |
| 1378 |
| 1379 When creating an insertion point one must care about the state that is |
| 1380 kept: |
| 1381 - formatting state (level, bol) is cloned and used in insertion points |
| 1382 as well |
| 1383 - labels, temps, exc_vars: One must construct a scope in which these can |
| 1384 exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for |
| 1385 sanity checking and forward compatabilty). Created insertion points |
| 1386 looses this scope and cannot access it. |
| 1387 - marker: Not copied to insertion point |
| 1388 - filename_table, filename_list, input_file_contents: All codewriters |
| 1389 coming from the same root share the same instances simultaneously. |
| 1390 """ |
| 1391 |
| 1392 # f file output file |
| 1393 # buffer StringIOTree |
| 1394 |
| 1395 # level int indentation level |
| 1396 # bol bool beginning of line? |
| 1397 # marker string comment to emit before next line |
| 1398 # funcstate FunctionState contains state local to a C function u
sed for code |
| 1399 # generation (labels and temps state etc
.) |
| 1400 # globalstate GlobalState contains state global for a C file (in
put file info, |
| 1401 # utility code, declared constants etc.) |
| 1402 # emit_linenums boolean whether or not to write #line pragmas |
| 1403 # |
| 1404 # c_line_in_traceback boolean append the c file and line number to t
he traceback for exceptions |
| 1405 # |
| 1406 # pyclass_stack list used during recursive code generation
to pass information |
| 1407 # about the current class one is in |
| 1408 |
| 1409 globalstate = None |
| 1410 |
| 1411 def __init__(self, create_from=None, buffer=None, copy_formatting=False, emi
t_linenums=None, c_line_in_traceback=True): |
| 1412 if buffer is None: buffer = StringIOTree() |
| 1413 self.buffer = buffer |
| 1414 self.marker = None |
| 1415 self.last_marker_line = 0 |
| 1416 self.source_desc = "" |
| 1417 self.pyclass_stack = [] |
| 1418 |
| 1419 self.funcstate = None |
| 1420 self.level = 0 |
| 1421 self.call_level = 0 |
| 1422 self.bol = 1 |
| 1423 |
| 1424 if create_from is not None: |
| 1425 # Use same global state |
| 1426 self.globalstate = create_from.globalstate |
| 1427 self.funcstate = create_from.funcstate |
| 1428 # Clone formatting state |
| 1429 if copy_formatting: |
| 1430 self.level = create_from.level |
| 1431 self.bol = create_from.bol |
| 1432 self.call_level = create_from.call_level |
| 1433 |
| 1434 if emit_linenums is None and self.globalstate: |
| 1435 self.emit_linenums = self.globalstate.emit_linenums |
| 1436 else: |
| 1437 self.emit_linenums = emit_linenums |
| 1438 self.c_line_in_traceback = c_line_in_traceback |
| 1439 |
| 1440 def create_new(self, create_from, buffer, copy_formatting): |
| 1441 # polymorphic constructor -- very slightly more versatile |
| 1442 # than using __class__ |
| 1443 result = CCodeWriter(create_from, buffer, copy_formatting, |
| 1444 c_line_in_traceback=self.c_line_in_traceback) |
| 1445 return result |
| 1446 |
| 1447 def copyto(self, f): |
| 1448 self.buffer.copyto(f) |
| 1449 |
| 1450 def getvalue(self): |
| 1451 return self.buffer.getvalue() |
| 1452 |
| 1453 def write(self, s): |
| 1454 # also put invalid markers (lineno 0), to indicate that those lines |
| 1455 # have no Cython source code correspondence |
| 1456 if self.marker is None: |
| 1457 cython_lineno = self.last_marker_line |
| 1458 else: |
| 1459 cython_lineno = self.marker[0] |
| 1460 |
| 1461 self.buffer.markers.extend([cython_lineno] * s.count('\n')) |
| 1462 self.buffer.write(s) |
| 1463 |
| 1464 def insertion_point(self): |
| 1465 other = self.create_new(create_from=self, buffer=self.buffer.insertion_p
oint(), copy_formatting=True) |
| 1466 return other |
| 1467 |
| 1468 def new_writer(self): |
| 1469 """ |
| 1470 Creates a new CCodeWriter connected to the same global state, which |
| 1471 can later be inserted using insert. |
| 1472 """ |
| 1473 return CCodeWriter(create_from=self, c_line_in_traceback=self.c_line_in_
traceback) |
| 1474 |
| 1475 def insert(self, writer): |
| 1476 """ |
| 1477 Inserts the contents of another code writer (created with |
| 1478 the same global state) in the current location. |
| 1479 |
| 1480 It is ok to write to the inserted writer also after insertion. |
| 1481 """ |
| 1482 assert writer.globalstate is self.globalstate |
| 1483 self.buffer.insert(writer.buffer) |
| 1484 |
| 1485 # Properties delegated to function scope |
| 1486 label_counter = funccontext_property("label_counter") |
| 1487 return_label = funccontext_property("return_label") |
| 1488 error_label = funccontext_property("error_label") |
| 1489 labels_used = funccontext_property("labels_used") |
| 1490 continue_label = funccontext_property("continue_label") |
| 1491 break_label = funccontext_property("break_label") |
| 1492 return_from_error_cleanup_label = funccontext_property("return_from_error_cl
eanup_label") |
| 1493 yield_labels = funccontext_property("yield_labels") |
| 1494 |
| 1495 # Functions delegated to function scope |
| 1496 def new_label(self, name=None): return self.funcstate.new_label(name) |
| 1497 def new_error_label(self): return self.funcstate.new_error_label() |
| 1498 def new_yield_label(self): return self.funcstate.new_yield_label() |
| 1499 def get_loop_labels(self): return self.funcstate.get_loop_labels() |
| 1500 def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(lab
els) |
| 1501 def new_loop_labels(self): return self.funcstate.new_loop_labels() |
| 1502 def get_all_labels(self): return self.funcstate.get_all_labels() |
| 1503 def set_all_labels(self, labels): return self.funcstate.set_all_labels(labe
ls) |
| 1504 def all_new_labels(self): return self.funcstate.all_new_labels() |
| 1505 def use_label(self, lbl): return self.funcstate.use_label(lbl) |
| 1506 def label_used(self, lbl): return self.funcstate.label_used(lbl) |
| 1507 |
| 1508 |
| 1509 def enter_cfunc_scope(self): |
| 1510 self.funcstate = FunctionState(self) |
| 1511 |
| 1512 def exit_cfunc_scope(self): |
| 1513 self.funcstate = None |
| 1514 |
| 1515 # constant handling |
| 1516 |
| 1517 def get_py_int(self, str_value, longness): |
| 1518 return self.globalstate.get_int_const(str_value, longness).cname |
| 1519 |
| 1520 def get_py_float(self, str_value, value_code): |
| 1521 return self.globalstate.get_float_const(str_value, value_code).cname |
| 1522 |
| 1523 def get_py_const(self, type, prefix='', cleanup_level=None): |
| 1524 return self.globalstate.get_py_const(type, prefix, cleanup_level).cname |
| 1525 |
| 1526 def get_string_const(self, text): |
| 1527 return self.globalstate.get_string_const(text).cname |
| 1528 |
| 1529 def get_pyunicode_ptr_const(self, text): |
| 1530 return self.globalstate.get_pyunicode_ptr_const(text) |
| 1531 |
| 1532 def get_py_string_const(self, text, identifier=None, |
| 1533 is_str=False, unicode_value=None): |
| 1534 return self.globalstate.get_py_string_const( |
| 1535 text, identifier, is_str, unicode_value).cname |
| 1536 |
| 1537 def get_argument_default_const(self, type): |
| 1538 return self.globalstate.get_py_const(type).cname |
| 1539 |
| 1540 def intern(self, text): |
| 1541 return self.get_py_string_const(text) |
| 1542 |
| 1543 def intern_identifier(self, text): |
| 1544 return self.get_py_string_const(text, identifier=True) |
| 1545 |
| 1546 def get_cached_constants_writer(self): |
| 1547 return self.globalstate.get_cached_constants_writer() |
| 1548 |
| 1549 # code generation |
| 1550 |
| 1551 def putln(self, code="", safe=False): |
| 1552 if self.marker and self.bol: |
| 1553 self.emit_marker() |
| 1554 if self.emit_linenums and self.last_marker_line != 0: |
| 1555 self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source
_desc)) |
| 1556 |
| 1557 if code: |
| 1558 if safe: |
| 1559 self.put_safe(code) |
| 1560 else: |
| 1561 self.put(code) |
| 1562 self.write("\n") |
| 1563 self.bol = 1 |
| 1564 |
| 1565 def emit_marker(self): |
| 1566 self.write("\n") |
| 1567 self.indent() |
| 1568 self.write("/* %s */\n" % self.marker[1]) |
| 1569 if (self.funcstate and self.funcstate.can_trace |
| 1570 and self.globalstate.directives['linetrace']): |
| 1571 self.indent() |
| 1572 self.write('__Pyx_TraceLine(%d)\n' % self.marker[0]) |
| 1573 self.last_marker_line = self.marker[0] |
| 1574 self.marker = None |
| 1575 |
| 1576 def put_safe(self, code): |
| 1577 # put code, but ignore {} |
| 1578 self.write(code) |
| 1579 self.bol = 0 |
| 1580 |
| 1581 def put_or_include(self, code, name): |
| 1582 include_dir = self.globalstate.common_utility_include_dir |
| 1583 if include_dir and len(code) > 1024: |
| 1584 include_file = "%s_%s.h" % ( |
| 1585 name, hashlib.md5(code.encode('utf8')).hexdigest()) |
| 1586 path = os.path.join(include_dir, include_file) |
| 1587 if not os.path.exists(path): |
| 1588 tmp_path = '%s.tmp%s' % (path, os.getpid()) |
| 1589 f = Utils.open_new_file(tmp_path) |
| 1590 try: |
| 1591 f.write(code) |
| 1592 finally: |
| 1593 f.close() |
| 1594 os.rename(tmp_path, path) |
| 1595 code = '#include "%s"\n' % path |
| 1596 self.put(code) |
| 1597 |
| 1598 def put(self, code): |
| 1599 if is_self_assignment(code): |
| 1600 return |
| 1601 fix_indent = False |
| 1602 if "{" in code: |
| 1603 dl = code.count("{") |
| 1604 else: |
| 1605 dl = 0 |
| 1606 if "}" in code: |
| 1607 dl -= code.count("}") |
| 1608 if dl < 0: |
| 1609 self.level += dl |
| 1610 elif dl == 0 and code[0] == "}": |
| 1611 # special cases like "} else {" need a temporary dedent |
| 1612 fix_indent = True |
| 1613 self.level -= 1 |
| 1614 if self.bol: |
| 1615 self.indent() |
| 1616 self.write(code) |
| 1617 self.bol = 0 |
| 1618 if dl > 0: |
| 1619 self.level += dl |
| 1620 elif fix_indent: |
| 1621 self.level += 1 |
| 1622 |
| 1623 def putln_tempita(self, code, **context): |
| 1624 from Cython.Tempita import sub |
| 1625 self.putln(sub(code, **context)) |
| 1626 |
| 1627 def put_tempita(self, code, **context): |
| 1628 from Cython.Tempita import sub |
| 1629 self.put(sub(code, **context)) |
| 1630 |
| 1631 def increase_indent(self): |
| 1632 self.level += 1 |
| 1633 |
| 1634 def decrease_indent(self): |
| 1635 self.level -= 1 |
| 1636 |
| 1637 def begin_block(self): |
| 1638 self.putln("{") |
| 1639 self.increase_indent() |
| 1640 |
| 1641 def end_block(self): |
| 1642 self.decrease_indent() |
| 1643 self.putln("}") |
| 1644 |
| 1645 def indent(self): |
| 1646 self.write(" " * self.level) |
| 1647 |
| 1648 def get_py_version_hex(self, pyversion): |
| 1649 return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4] |
| 1650 |
| 1651 def mark_pos(self, pos): |
| 1652 if pos is None: |
| 1653 return |
| 1654 source_desc, line, col = pos |
| 1655 if self.last_marker_line == line: |
| 1656 return |
| 1657 assert isinstance(source_desc, SourceDescriptor) |
| 1658 contents = self.globalstate.commented_file_contents(source_desc) |
| 1659 lines = contents[max(0, line-3):line] # line numbers start at 1 |
| 1660 lines[-1] += u' # <<<<<<<<<<<<<<' |
| 1661 lines += contents[line:line+2] |
| 1662 |
| 1663 marker = u'"%s":%d\n%s\n' % ( |
| 1664 source_desc.get_escaped_description(), line, u'\n'.join(lines)) |
| 1665 self.marker = (line, marker) |
| 1666 if self.emit_linenums: |
| 1667 self.source_desc = source_desc.get_escaped_description() |
| 1668 |
| 1669 def put_label(self, lbl): |
| 1670 if lbl in self.funcstate.labels_used: |
| 1671 self.putln("%s:;" % lbl) |
| 1672 |
| 1673 def put_goto(self, lbl): |
| 1674 self.funcstate.use_label(lbl) |
| 1675 self.putln("goto %s;" % lbl) |
| 1676 |
| 1677 def put_var_declaration(self, entry, storage_class="", |
| 1678 dll_linkage=None, definition=True): |
| 1679 #print "Code.put_var_declaration:", entry.name, "definition =", definiti
on ### |
| 1680 if entry.visibility == 'private' and not (definition or entry.defined_in
_pxd): |
| 1681 #print "...private and not definition, skipping", entry.cname ### |
| 1682 return |
| 1683 if entry.visibility == "private" and not entry.used: |
| 1684 #print "...private and not used, skipping", entry.cname ### |
| 1685 return |
| 1686 if storage_class: |
| 1687 self.put("%s " % storage_class) |
| 1688 if not entry.cf_used: |
| 1689 self.put('CYTHON_UNUSED ') |
| 1690 self.put(entry.type.declaration_code( |
| 1691 entry.cname, dll_linkage=dll_linkage)) |
| 1692 if entry.init is not None: |
| 1693 self.put_safe(" = %s" % entry.type.literal_code(entry.init)) |
| 1694 elif entry.type.is_pyobject: |
| 1695 self.put(" = NULL") |
| 1696 self.putln(";") |
| 1697 |
| 1698 def put_temp_declarations(self, func_context): |
| 1699 for name, type, manage_ref, static in func_context.temps_allocated: |
| 1700 decl = type.declaration_code(name) |
| 1701 if type.is_pyobject: |
| 1702 self.putln("%s = NULL;" % decl) |
| 1703 elif type.is_memoryviewslice: |
| 1704 import MemoryView |
| 1705 self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) |
| 1706 else: |
| 1707 self.putln("%s%s;" % (static and "static " or "", decl)) |
| 1708 |
| 1709 if func_context.should_declare_error_indicator: |
| 1710 if self.funcstate.uses_error_indicator: |
| 1711 unused = '' |
| 1712 else: |
| 1713 unused = 'CYTHON_UNUSED ' |
| 1714 # Initialize these variables to silence compiler warnings |
| 1715 self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname)) |
| 1716 self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cna
me)) |
| 1717 self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname)) |
| 1718 |
| 1719 def put_h_guard(self, guard): |
| 1720 self.putln("#ifndef %s" % guard) |
| 1721 self.putln("#define %s" % guard) |
| 1722 |
| 1723 def unlikely(self, cond): |
| 1724 if Options.gcc_branch_hints: |
| 1725 return 'unlikely(%s)' % cond |
| 1726 else: |
| 1727 return cond |
| 1728 |
| 1729 def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper)
: |
| 1730 if not modifiers: |
| 1731 return '' |
| 1732 return '%s ' % ' '.join([mapper(m,m) for m in modifiers]) |
| 1733 |
| 1734 # Python objects and reference counting |
| 1735 |
| 1736 def entry_as_pyobject(self, entry): |
| 1737 type = entry.type |
| 1738 if (not entry.is_self_arg and not entry.type.is_complete() |
| 1739 or entry.type.is_extension_type): |
| 1740 return "(PyObject *)" + entry.cname |
| 1741 else: |
| 1742 return entry.cname |
| 1743 |
| 1744 def as_pyobject(self, cname, type): |
| 1745 from PyrexTypes import py_object_type, typecast |
| 1746 return typecast(py_object_type, type, cname) |
| 1747 |
| 1748 def put_gotref(self, cname): |
| 1749 self.putln("__Pyx_GOTREF(%s);" % cname) |
| 1750 |
| 1751 def put_giveref(self, cname): |
| 1752 self.putln("__Pyx_GIVEREF(%s);" % cname) |
| 1753 |
| 1754 def put_xgiveref(self, cname): |
| 1755 self.putln("__Pyx_XGIVEREF(%s);" % cname) |
| 1756 |
| 1757 def put_xgotref(self, cname): |
| 1758 self.putln("__Pyx_XGOTREF(%s);" % cname) |
| 1759 |
| 1760 def put_incref(self, cname, type, nanny=True): |
| 1761 if nanny: |
| 1762 self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) |
| 1763 else: |
| 1764 self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) |
| 1765 |
| 1766 def put_decref(self, cname, type, nanny=True): |
| 1767 self._put_decref(cname, type, nanny, null_check=False, clear=False) |
| 1768 |
| 1769 def put_var_gotref(self, entry): |
| 1770 if entry.type.is_pyobject: |
| 1771 self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) |
| 1772 |
| 1773 def put_var_giveref(self, entry): |
| 1774 if entry.type.is_pyobject: |
| 1775 self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) |
| 1776 |
| 1777 def put_var_xgotref(self, entry): |
| 1778 if entry.type.is_pyobject: |
| 1779 self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) |
| 1780 |
| 1781 def put_var_xgiveref(self, entry): |
| 1782 if entry.type.is_pyobject: |
| 1783 self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) |
| 1784 |
| 1785 def put_var_incref(self, entry): |
| 1786 if entry.type.is_pyobject: |
| 1787 self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) |
| 1788 |
| 1789 def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=Fals
e): |
| 1790 self._put_decref(cname, type, nanny, null_check=False, |
| 1791 clear=True, clear_before_decref=clear_before_decref) |
| 1792 |
| 1793 def put_xdecref(self, cname, type, nanny=True, have_gil=True): |
| 1794 self._put_decref(cname, type, nanny, null_check=True, |
| 1795 have_gil=have_gil, clear=False) |
| 1796 |
| 1797 def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=Fal
se): |
| 1798 self._put_decref(cname, type, nanny, null_check=True, |
| 1799 clear=True, clear_before_decref=clear_before_decref) |
| 1800 |
| 1801 def _put_decref(self, cname, type, nanny=True, null_check=False, |
| 1802 have_gil=True, clear=False, clear_before_decref=False): |
| 1803 if type.is_memoryviewslice: |
| 1804 self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) |
| 1805 return |
| 1806 |
| 1807 prefix = nanny and '__Pyx' or 'Py' |
| 1808 X = null_check and 'X' or '' |
| 1809 |
| 1810 if clear: |
| 1811 if clear_before_decref: |
| 1812 if not nanny: |
| 1813 X = '' # CPython doesn't have a Py_XCLEAR() |
| 1814 self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) |
| 1815 else: |
| 1816 self.putln("%s_%sDECREF(%s); %s = 0;" % ( |
| 1817 prefix, X, self.as_pyobject(cname, type), cname)) |
| 1818 else: |
| 1819 self.putln("%s_%sDECREF(%s);" % ( |
| 1820 prefix, X, self.as_pyobject(cname, type))) |
| 1821 |
| 1822 def put_decref_set(self, cname, rhs_cname): |
| 1823 self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) |
| 1824 |
| 1825 def put_xdecref_set(self, cname, rhs_cname): |
| 1826 self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) |
| 1827 |
| 1828 def put_var_decref(self, entry): |
| 1829 if entry.type.is_pyobject: |
| 1830 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) |
| 1831 |
| 1832 def put_var_xdecref(self, entry): |
| 1833 if entry.type.is_pyobject: |
| 1834 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) |
| 1835 |
| 1836 def put_var_decref_clear(self, entry): |
| 1837 self._put_var_decref_clear(entry, null_check=False) |
| 1838 |
| 1839 def put_var_xdecref_clear(self, entry): |
| 1840 self._put_var_decref_clear(entry, null_check=True) |
| 1841 |
| 1842 def _put_var_decref_clear(self, entry, null_check): |
| 1843 if entry.type.is_pyobject: |
| 1844 if entry.in_closure: |
| 1845 # reset before DECREF to make sure closure state is |
| 1846 # consistent during call to DECREF() |
| 1847 self.putln("__Pyx_%sCLEAR(%s);" % ( |
| 1848 null_check and 'X' or '', |
| 1849 entry.cname)) |
| 1850 else: |
| 1851 self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( |
| 1852 null_check and 'X' or '', |
| 1853 self.entry_as_pyobject(entry), |
| 1854 entry.cname)) |
| 1855 |
| 1856 def put_var_decrefs(self, entries, used_only = 0): |
| 1857 for entry in entries: |
| 1858 if not used_only or entry.used: |
| 1859 if entry.xdecref_cleanup: |
| 1860 self.put_var_xdecref(entry) |
| 1861 else: |
| 1862 self.put_var_decref(entry) |
| 1863 |
| 1864 def put_var_xdecrefs(self, entries): |
| 1865 for entry in entries: |
| 1866 self.put_var_xdecref(entry) |
| 1867 |
| 1868 def put_var_xdecrefs_clear(self, entries): |
| 1869 for entry in entries: |
| 1870 self.put_var_xdecref_clear(entry) |
| 1871 |
| 1872 def put_incref_memoryviewslice(self, slice_cname, have_gil=False): |
| 1873 import MemoryView |
| 1874 self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) |
| 1875 self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) |
| 1876 |
| 1877 def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): |
| 1878 import MemoryView |
| 1879 self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) |
| 1880 self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))
) |
| 1881 |
| 1882 def put_xgiveref_memoryviewslice(self, slice_cname): |
| 1883 self.put_xgiveref("%s.memview" % slice_cname) |
| 1884 |
| 1885 def put_init_to_py_none(self, cname, type, nanny=True): |
| 1886 from PyrexTypes import py_object_type, typecast |
| 1887 py_none = typecast(type, py_object_type, "Py_None") |
| 1888 if nanny: |
| 1889 self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none)) |
| 1890 else: |
| 1891 self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none)) |
| 1892 |
| 1893 def put_init_var_to_py_none(self, entry, template = "%s", nanny=True): |
| 1894 code = template % entry.cname |
| 1895 #if entry.type.is_extension_type: |
| 1896 # code = "((PyObject*)%s)" % code |
| 1897 self.put_init_to_py_none(code, entry.type, nanny) |
| 1898 if entry.in_closure: |
| 1899 self.put_giveref('Py_None') |
| 1900 |
| 1901 def put_pymethoddef(self, entry, term, allow_skip=True): |
| 1902 if entry.is_special or entry.name == '__getattribute__': |
| 1903 if entry.name not in ['__cinit__', '__dealloc__', '__richcmp__', '__
next__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', '__getcha
rbuffer__', '__getbuffer__', '__releasebuffer__']: |
| 1904 if entry.name == '__getattr__' and not self.globalstate.directiv
es['fast_getattr']: |
| 1905 pass |
| 1906 # Python's typeobject.c will automatically fill in our slot |
| 1907 # in add_operators() (called by PyType_Ready) with a value |
| 1908 # that's better than ours. |
| 1909 elif allow_skip: |
| 1910 return |
| 1911 from TypeSlots import method_coexist |
| 1912 if entry.doc: |
| 1913 doc_code = entry.doc_cname |
| 1914 else: |
| 1915 doc_code = 0 |
| 1916 method_flags = entry.signature.method_flags() |
| 1917 if method_flags: |
| 1918 if entry.is_special: |
| 1919 method_flags += [method_coexist] |
| 1920 self.putln( |
| 1921 '{__Pyx_NAMESTR("%s"), (PyCFunction)%s, %s, __Pyx_DOCSTR(%s)}%s'
% ( |
| 1922 entry.name, |
| 1923 entry.func_cname, |
| 1924 "|".join(method_flags), |
| 1925 doc_code, |
| 1926 term)) |
| 1927 |
| 1928 # GIL methods |
| 1929 |
| 1930 def put_ensure_gil(self, declare_gilstate=True, variable=None): |
| 1931 """ |
| 1932 Acquire the GIL. The generated code is safe even when no PyThreadState |
| 1933 has been allocated for this thread (for threads not initialized by |
| 1934 using the Python API). Additionally, the code generated by this method |
| 1935 may be called recursively. |
| 1936 """ |
| 1937 self.globalstate.use_utility_code( |
| 1938 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) |
| 1939 self.putln("#ifdef WITH_THREAD") |
| 1940 if not variable: |
| 1941 variable = '__pyx_gilstate_save' |
| 1942 if declare_gilstate: |
| 1943 self.put("PyGILState_STATE ") |
| 1944 self.putln("%s = PyGILState_Ensure();" % variable) |
| 1945 self.putln("#endif") |
| 1946 |
| 1947 def put_release_ensured_gil(self, variable=None): |
| 1948 """ |
| 1949 Releases the GIL, corresponds to `put_ensure_gil`. |
| 1950 """ |
| 1951 if not variable: |
| 1952 variable = '__pyx_gilstate_save' |
| 1953 self.putln("#ifdef WITH_THREAD") |
| 1954 self.putln("PyGILState_Release(%s);" % variable) |
| 1955 self.putln("#endif") |
| 1956 |
| 1957 def put_acquire_gil(self, variable=None): |
| 1958 """ |
| 1959 Acquire the GIL. The thread's thread state must have been initialized |
| 1960 by a previous `put_release_gil` |
| 1961 """ |
| 1962 self.putln("#ifdef WITH_THREAD") |
| 1963 if variable: |
| 1964 self.putln('_save = %s;' % variable) |
| 1965 self.putln("Py_BLOCK_THREADS") |
| 1966 self.putln("#endif") |
| 1967 |
| 1968 def put_release_gil(self, variable=None): |
| 1969 "Release the GIL, corresponds to `put_acquire_gil`." |
| 1970 self.putln("#ifdef WITH_THREAD") |
| 1971 self.putln("PyThreadState *_save;") |
| 1972 self.putln("Py_UNBLOCK_THREADS") |
| 1973 if variable: |
| 1974 self.putln('%s = _save;' % variable) |
| 1975 self.putln("#endif") |
| 1976 |
| 1977 def declare_gilstate(self): |
| 1978 self.putln("#ifdef WITH_THREAD") |
| 1979 self.putln("PyGILState_STATE __pyx_gilstate_save;") |
| 1980 self.putln("#endif") |
| 1981 |
| 1982 # error handling |
| 1983 |
| 1984 def put_error_if_neg(self, pos, value): |
| 1985 # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(
pos))) # TODO this path is almost _never_ taken, yet this macro makes is slower
! |
| 1986 return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) |
| 1987 |
| 1988 def put_error_if_unbound(self, pos, entry, in_nogil_context=False): |
| 1989 import ExprNodes |
| 1990 if entry.from_closure: |
| 1991 func = '__Pyx_RaiseClosureNameError' |
| 1992 self.globalstate.use_utility_code( |
| 1993 ExprNodes.raise_closure_name_error_utility_code) |
| 1994 elif entry.type.is_memoryviewslice and in_nogil_context: |
| 1995 func = '__Pyx_RaiseUnboundMemoryviewSliceNogil' |
| 1996 self.globalstate.use_utility_code( |
| 1997 ExprNodes.raise_unbound_memoryview_utility_code_nogil) |
| 1998 else: |
| 1999 func = '__Pyx_RaiseUnboundLocalError' |
| 2000 self.globalstate.use_utility_code( |
| 2001 ExprNodes.raise_unbound_local_error_utility_code) |
| 2002 |
| 2003 self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % ( |
| 2004 entry.type.check_for_null_code(entry.cname), |
| 2005 func, |
| 2006 entry.name, |
| 2007 self.error_goto(pos))) |
| 2008 |
| 2009 def set_error_info(self, pos, used=False): |
| 2010 self.funcstate.should_declare_error_indicator = True |
| 2011 if used: |
| 2012 self.funcstate.uses_error_indicator = True |
| 2013 if self.c_line_in_traceback: |
| 2014 cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro) |
| 2015 else: |
| 2016 cinfo = "" |
| 2017 |
| 2018 return "%s = %s[%s]; %s = %s;%s" % ( |
| 2019 Naming.filename_cname, |
| 2020 Naming.filetable_cname, |
| 2021 self.lookup_filename(pos[0]), |
| 2022 Naming.lineno_cname, |
| 2023 pos[1], |
| 2024 cinfo) |
| 2025 |
| 2026 def error_goto(self, pos): |
| 2027 lbl = self.funcstate.error_label |
| 2028 self.funcstate.use_label(lbl) |
| 2029 return "{%s goto %s;}" % ( |
| 2030 self.set_error_info(pos), |
| 2031 lbl) |
| 2032 |
| 2033 def error_goto_if(self, cond, pos): |
| 2034 return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos)) |
| 2035 |
| 2036 def error_goto_if_null(self, cname, pos): |
| 2037 return self.error_goto_if("!%s" % cname, pos) |
| 2038 |
| 2039 def error_goto_if_neg(self, cname, pos): |
| 2040 return self.error_goto_if("%s < 0" % cname, pos) |
| 2041 |
| 2042 def error_goto_if_PyErr(self, pos): |
| 2043 return self.error_goto_if("PyErr_Occurred()", pos) |
| 2044 |
| 2045 def lookup_filename(self, filename): |
| 2046 return self.globalstate.lookup_filename(filename) |
| 2047 |
| 2048 def put_declare_refcount_context(self): |
| 2049 self.putln('__Pyx_RefNannyDeclarations') |
| 2050 |
| 2051 def put_setup_refcount_context(self, name, acquire_gil=False): |
| 2052 if acquire_gil: |
| 2053 self.globalstate.use_utility_code( |
| 2054 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")
) |
| 2055 self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil
and 1 or 0)) |
| 2056 |
| 2057 def put_finish_refcount_context(self): |
| 2058 self.putln("__Pyx_RefNannyFinishContext();") |
| 2059 |
| 2060 def put_add_traceback(self, qualified_name): |
| 2061 """ |
| 2062 Build a Python traceback for propagating exceptions. |
| 2063 |
| 2064 qualified_name should be the qualified name of the function. |
| 2065 """ |
| 2066 format_tuple = ( |
| 2067 qualified_name, |
| 2068 Naming.clineno_cname, |
| 2069 Naming.lineno_cname, |
| 2070 Naming.filename_cname, |
| 2071 ) |
| 2072 self.funcstate.uses_error_indicator = True |
| 2073 self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) |
| 2074 |
| 2075 def put_unraisable(self, qualified_name): |
| 2076 """ |
| 2077 Generate code to print a Python warning for an unraisable exception. |
| 2078 |
| 2079 qualified_name should be the qualified name of the function. |
| 2080 """ |
| 2081 format_tuple = ( |
| 2082 qualified_name, |
| 2083 Naming.clineno_cname, |
| 2084 Naming.lineno_cname, |
| 2085 Naming.filename_cname, |
| 2086 int(self.globalstate.directives['unraisable_tracebacks']) |
| 2087 ) |
| 2088 self.funcstate.uses_error_indicator = True |
| 2089 self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %s);' % format_tuple
) |
| 2090 self.globalstate.use_utility_code( |
| 2091 UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c")) |
| 2092 |
| 2093 def put_trace_declarations(self): |
| 2094 self.putln('__Pyx_TraceDeclarations') |
| 2095 |
| 2096 def put_trace_call(self, name, pos): |
| 2097 self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetabl
e_cname, self.lookup_filename(pos[0]), pos[1])) |
| 2098 |
| 2099 def put_trace_exception(self): |
| 2100 self.putln("__Pyx_TraceException();") |
| 2101 |
| 2102 def put_trace_return(self, retvalue_cname): |
| 2103 self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname) |
| 2104 |
| 2105 def putln_openmp(self, string): |
| 2106 self.putln("#ifdef _OPENMP") |
| 2107 self.putln(string) |
| 2108 self.putln("#endif /* _OPENMP */") |
| 2109 |
| 2110 def undef_builtin_expect(self, cond): |
| 2111 """ |
| 2112 Redefine the macros likely() and unlikely to no-ops, depending on |
| 2113 condition 'cond' |
| 2114 """ |
| 2115 self.putln("#if %s" % cond) |
| 2116 self.putln(" #undef likely") |
| 2117 self.putln(" #undef unlikely") |
| 2118 self.putln(" #define likely(x) (x)") |
| 2119 self.putln(" #define unlikely(x) (x)") |
| 2120 self.putln("#endif") |
| 2121 |
| 2122 def redef_builtin_expect(self, cond): |
| 2123 self.putln("#if %s" % cond) |
| 2124 self.putln(" #undef likely") |
| 2125 self.putln(" #undef unlikely") |
| 2126 self.putln(" #define likely(x) __builtin_expect(!!(x), 1)") |
| 2127 self.putln(" #define unlikely(x) __builtin_expect(!!(x), 0)") |
| 2128 self.putln("#endif") |
| 2129 |
| 2130 class PyrexCodeWriter(object): |
| 2131 # f file output file |
| 2132 # level int indentation level |
| 2133 |
| 2134 def __init__(self, outfile_name): |
| 2135 self.f = Utils.open_new_file(outfile_name) |
| 2136 self.level = 0 |
| 2137 |
| 2138 def putln(self, code): |
| 2139 self.f.write("%s%s\n" % (" " * self.level, code)) |
| 2140 |
| 2141 def indent(self): |
| 2142 self.level += 1 |
| 2143 |
| 2144 def dedent(self): |
| 2145 self.level -= 1 |
| 2146 |
| 2147 class PyxCodeWriter(object): |
| 2148 """ |
| 2149 Can be used for writing out some Cython code. To use the indenter |
| 2150 functionality, the Cython.Compiler.Importer module will have to be used |
| 2151 to load the code to support python 2.4 |
| 2152 """ |
| 2153 |
| 2154 def __init__(self, buffer=None, indent_level=0, context=None, encoding='asci
i'): |
| 2155 self.buffer = buffer or StringIOTree() |
| 2156 self.level = indent_level |
| 2157 self.context = context |
| 2158 self.encoding = encoding |
| 2159 |
| 2160 def indent(self, levels=1): |
| 2161 self.level += levels |
| 2162 return True |
| 2163 |
| 2164 def dedent(self, levels=1): |
| 2165 self.level -= levels |
| 2166 |
| 2167 def indenter(self, line): |
| 2168 """ |
| 2169 Instead of |
| 2170 |
| 2171 with pyx_code.indenter("for i in range(10):"): |
| 2172 pyx_code.putln("print i") |
| 2173 |
| 2174 write |
| 2175 |
| 2176 if pyx_code.indenter("for i in range(10);"): |
| 2177 pyx_code.putln("print i") |
| 2178 pyx_code.dedent() |
| 2179 """ |
| 2180 self.putln(line) |
| 2181 self.indent() |
| 2182 return True |
| 2183 |
| 2184 def getvalue(self): |
| 2185 result = self.buffer.getvalue() |
| 2186 if not isinstance(result, unicode): |
| 2187 result = result.decode(self.encoding) |
| 2188 |
| 2189 return result |
| 2190 |
| 2191 def putln(self, line, context=None): |
| 2192 context = context or self.context |
| 2193 if context: |
| 2194 line = sub_tempita(line, context) |
| 2195 self._putln(line) |
| 2196 |
| 2197 def _putln(self, line): |
| 2198 self.buffer.write("%s%s\n" % (self.level * " ", line)) |
| 2199 |
| 2200 def put_chunk(self, chunk, context=None): |
| 2201 context = context or self.context |
| 2202 if context: |
| 2203 chunk = sub_tempita(chunk, context) |
| 2204 |
| 2205 chunk = textwrap.dedent(chunk) |
| 2206 for line in chunk.splitlines(): |
| 2207 self._putln(line) |
| 2208 |
| 2209 def insertion_point(self): |
| 2210 return PyxCodeWriter(self.buffer.insertion_point(), self.level, |
| 2211 self.context) |
| 2212 |
| 2213 def named_insertion_point(self, name): |
| 2214 setattr(self, name, self.insertion_point()) |
| 2215 |
| 2216 |
| 2217 class ClosureTempAllocator(object): |
| 2218 def __init__(self, klass): |
| 2219 self.klass = klass |
| 2220 self.temps_allocated = {} |
| 2221 self.temps_free = {} |
| 2222 self.temps_count = 0 |
| 2223 |
| 2224 def reset(self): |
| 2225 for type, cnames in self.temps_allocated.items(): |
| 2226 self.temps_free[type] = list(cnames) |
| 2227 |
| 2228 def allocate_temp(self, type): |
| 2229 if not type in self.temps_allocated: |
| 2230 self.temps_allocated[type] = [] |
| 2231 self.temps_free[type] = [] |
| 2232 elif self.temps_free[type]: |
| 2233 return self.temps_free[type].pop(0) |
| 2234 cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count) |
| 2235 self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_
cdef=True) |
| 2236 self.temps_allocated[type].append(cname) |
| 2237 self.temps_count += 1 |
| 2238 return cname |
OLD | NEW |