maintransformer.py 57.7 KB
Newer Older
Colin Walters's avatar
Colin Walters committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# -*- Mode: Python -*-
# Copyright (C) 2010 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
#

import re

from . import ast
23
from . import message
24
25
26
27
28
29
30
31
32
33
from .annotationparser import (TAG_DEPRECATED, TAG_SINCE, TAG_STABILITY, TAG_RETURNS)
from .annotationparser import (ANN_ALLOW_NONE, ANN_ARRAY, ANN_ATTRIBUTES, ANN_CLOSURE,
                               ANN_CONSTRUCTOR, ANN_DESTROY, ANN_ELEMENT_TYPE, ANN_FOREIGN,
                               ANN_GET_VALUE_FUNC, ANN_IN, ANN_INOUT, ANN_METHOD, ANN_OUT,
                               ANN_REF_FUNC, ANN_RENAME_TO, ANN_SCOPE, ANN_SET_VALUE_FUNC,
                               ANN_SKIP, ANN_TRANSFER, ANN_TYPE, ANN_UNREF_FUNC, ANN_VALUE,
                               ANN_VFUNC)
from .annotationparser import (OPT_ARRAY_FIXED_SIZE, OPT_ARRAY_LENGTH, OPT_ARRAY_ZERO_TERMINATED,
                               OPT_OUT_CALLEE_ALLOCATES, OPT_OUT_CALLER_ALLOCATES,
                               OPT_TRANSFER_FLOATING, OPT_TRANSFER_NONE)
34

35
from .utils import to_underscores_noprefix
Colin Walters's avatar
Colin Walters committed
36

37

Colin Walters's avatar
Colin Walters committed
38
39
40
41
42
43
44
45
46
47
48
class MainTransformer(object):

    def __init__(self, transformer, blocks):
        self._transformer = transformer
        self._blocks = blocks
        self._namespace = transformer.namespace
        self._uscore_type_names = {}

    # Public API

    def transform(self):
49
50
51
52
        if not self._namespace.names:
            message.fatal('Namespace is empty; likely causes are:\n'
                          '* Not including .h files to be scanned\n'
                          '* Broken --identifier-prefix')
53

54
55
56
        # Some initial namespace surgery
        self._namespace.walk(self._pass_fixup_hidden_fields)

Colin Walters's avatar
Colin Walters committed
57
58
59
60
61
        # We have a rough tree which should have most of of the types
        # we know about.  Let's attempt closure; walk over all of the
        # Type() types and see if they match up with something.
        self._namespace.walk(self._pass_type_resolution)

62
63
64
        # Read in annotations needed early
        self._namespace.walk(self._pass_read_annotations_early)

Colin Walters's avatar
Colin Walters committed
65
66
67
68
69
70
71
72
73
74
75
76
77
        # Determine some default values for transfer etc.
        # based on the current tree.
        self._namespace.walk(self._pass_callable_defaults)

        # Read in most annotations now.
        self._namespace.walk(self._pass_read_annotations)

        # Now that we've possibly seen more types from annotations,
        # do another type resolution pass.
        self._namespace.walk(self._pass_type_resolution)

        # Generate a reverse mapping "bar_baz" -> BarBaz
        for node in self._namespace.itervalues():
Colin Walters's avatar
Colin Walters committed
78
            if isinstance(node, ast.Registered) and node.get_type is not None:
Colin Walters's avatar
Colin Walters committed
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
                self._uscore_type_names[node.c_symbol_prefix] = node
            elif isinstance(node, (ast.Record, ast.Union)):
                uscored = to_underscores_noprefix(node.name).lower()
                self._uscore_type_names[uscored] = node

        for node in list(self._namespace.itervalues()):
            if isinstance(node, ast.Function):
                # Discover which toplevel functions are actually methods
                self._pair_function(node)
            if isinstance(node, (ast.Class, ast.Interface)):
                self._pair_class_virtuals(node)

        # Some annotations need to be post function pairing
        self._namespace.walk(self._pass_read_annotations2)

        # Another type resolution pass after we've parsed virtuals, etc.
        self._namespace.walk(self._pass_type_resolution)

        self._namespace.walk(self._pass3)

        # TODO - merge into pass3
Colin Walters's avatar
Colin Walters committed
100
        self._pair_quarks_with_enums()
Colin Walters's avatar
Colin Walters committed
101
102
103

    # Private

104
105
    def _pass_fixup_hidden_fields(self, node, chain):
        """Hide all callbacks starting with _; the typical
106
        usage is void (*_gtk_reserved1)(void);"""
107
108
109
        if isinstance(node, (ast.Class, ast.Interface, ast.Record, ast.Union)):
            for field in node.fields:
                if (field
110
                and field.name is not None
111
                and field.name.startswith('_')
112
113
                and field.anonymous_node is not None
                and isinstance(field.anonymous_node, ast.Callback)):
114
                    field.introspectable = False
115
116
        return True

Colin Walters's avatar
Colin Walters committed
117
118
119
    def _get_validate_parameter_name(self, parent, param_name, origin):
        try:
            param = parent.get_parameter(param_name)
120
        except ValueError:
Colin Walters's avatar
Colin Walters committed
121
122
123
124
125
126
            param = None
        if param is None:
            if isinstance(origin, ast.Parameter):
                origin_name = 'parameter %s' % (origin.argname, )
            else:
                origin_name = 'return value'
127
128
            message.log_node(
                message.FATAL, parent,
Colin Walters's avatar
Colin Walters committed
129
                "can't find parameter %s referenced by %s of %r"
130
                % (param_name, origin_name, parent.name))
Colin Walters's avatar
Colin Walters committed
131
132
133

        return param.argname

134
135
136
137
138
139
140
141
142
143
144
145
146
147
    def _get_validate_field_name(self, parent, field_name, origin):
        try:
            field = parent.get_field(field_name)
        except ValueError:
            field = None
        if field is None:
            origin_name = 'field %s' % (origin.name, )
            message.log_node(
                message.FATAL, parent,
                "can't find field %s referenced by %s of %r"
                % (field_name, origin_name, parent.name))

        return field.name

148
    def _apply_annotation_rename_to(self, node, chain, block):
Colin Walters's avatar
Colin Walters committed
149
150
        if not block:
            return
151
        rename_to = block.annotations.get(ANN_RENAME_TO)
152
153
        if not rename_to:
            return
154
        rename_to = rename_to[0]
Colin Walters's avatar
Colin Walters committed
155
156
        target = self._namespace.get_by_symbol(rename_to)
        if not target:
157
            message.warn_node(node,
158
                "Can't find symbol %r referenced by \"rename-to\" annotation" % (rename_to, ))
Colin Walters's avatar
Colin Walters committed
159
        elif target.shadowed_by:
160
            message.warn_node(node,
161
162
163
164
                "Function %r already shadowed by %r, can't overwrite "
                "with %r" % (target.symbol,
                             target.shadowed_by,
                             rename_to))
Colin Walters's avatar
Colin Walters committed
165
        elif target.shadows:
166
            message.warn_node(node,
167
168
169
170
                "Function %r already shadows %r, can't multiply shadow "
                "with %r" % (target.symbol,
                             target.shadows,
                             rename_to))
Colin Walters's avatar
Colin Walters committed
171
        else:
172
173
            target.shadowed_by = node.name
            node.shadows = target.name
Colin Walters's avatar
Colin Walters committed
174
175
176
177
178

    def _apply_annotations_function(self, node, chain):
        block = self._blocks.get(node.symbol)
        self._apply_annotations_callable(node, chain, block)

179
180
181
182
183
184
185
186
187
    def _pass_read_annotations_early(self, node, chain):
        if isinstance(node, ast.Record):
            if node.ctype is not None:
                block = self._blocks.get(node.ctype)
            else:
                block = self._blocks.get(node.c_name)
            self._apply_annotations_annotated(node, block)
        return True

Colin Walters's avatar
Colin Walters committed
188
    def _pass_callable_defaults(self, node, chain):
Colin Walters's avatar
Colin Walters committed
189
        if isinstance(node, (ast.Callable, ast.Signal)):
Colin Walters's avatar
Colin Walters committed
190
191
192
193
194
195
196
            for param in node.parameters:
                if param.transfer is None:
                    param.transfer = self._get_transfer_default(node, param)
            if node.retval.transfer is None:
                node.retval.transfer = self._get_transfer_default(node, node.retval)
        return True

197
198
199
    def _get_annotation_name(self, node):
        if isinstance(node, (ast.Class, ast.Interface, ast.Record,
                             ast.Union, ast.Enum, ast.Bitfield,
200
                             ast.Callback, ast.Alias, ast.Constant)):
201
202
203
204
205
            if node.ctype is not None:
                return node.ctype
            elif isinstance(node, ast.Registered) and node.gtype_name is not None:
                return node.gtype_name
            return node.c_name
206
        raise AssertionError("Unhandled node %r" % (node, ))
207
208
209
210

    def _get_block(self, node):
        return self._blocks.get(self._get_annotation_name(node))

Colin Walters's avatar
Colin Walters committed
211
212
213
    def _pass_read_annotations(self, node, chain):
        if not node.namespace:
            return False
214
215
        if isinstance(node, ast.Alias):
            self._apply_annotations_alias(node, chain)
Colin Walters's avatar
Colin Walters committed
216
217
218
        if isinstance(node, ast.Function):
            self._apply_annotations_function(node, chain)
        if isinstance(node, ast.Callback):
219
            self._apply_annotations_callable(node, chain, block=self._get_block(node))
220
221
        if isinstance(node, (ast.Class, ast.Interface, ast.Union, ast.Enum,
                             ast.Bitfield, ast.Callback)):
222
            self._apply_annotations_annotated(node, self._get_block(node))
223
224
        if isinstance(node, (ast.Enum, ast.Bitfield)):
            self._apply_annotations_enum_members(node, self._get_block(node))
Colin Walters's avatar
Colin Walters committed
225
        if isinstance(node, (ast.Class, ast.Interface, ast.Record, ast.Union)):
226
            block = self._get_block(node)
Colin Walters's avatar
Colin Walters committed
227
228
            for field in node.fields:
                self._apply_annotations_field(node, block, field)
Johan Dahlin's avatar
Johan Dahlin committed
229
            name = self._get_annotation_name(node)
230
            section_name = 'SECTION:%s' % (name.lower(), )
Johan Dahlin's avatar
Johan Dahlin committed
231
            block = self._blocks.get(section_name)
232
233
            if block and block.description:
                node.doc = block.description
Colin Walters's avatar
Colin Walters committed
234
235
236
237
238
239
        if isinstance(node, (ast.Class, ast.Interface)):
            for prop in node.properties:
                self._apply_annotations_property(node, prop)
            for sig in node.signals:
                self._apply_annotations_signal(node, sig)
        if isinstance(node, ast.Class):
240
            block = self._get_block(node)
Colin Walters's avatar
Colin Walters committed
241
            if block:
242
243
244
245
246
247
248
249
                annotation = block.annotations.get(ANN_UNREF_FUNC)
                node.unref_func = annotation[0] if annotation else None
                annotation = block.annotations.get(ANN_REF_FUNC)
                node.ref_func = annotation[0] if annotation else None
                annotation = block.annotations.get(ANN_SET_VALUE_FUNC)
                node.set_value_func = annotation[0] if annotation else None
                annotation = block.annotations.get(ANN_GET_VALUE_FUNC)
                node.get_value_func = annotation[0] if annotation else None
Johan Dahlin's avatar
Johan Dahlin committed
250
251
        if isinstance(node, ast.Constant):
            self._apply_annotations_constant(node)
Colin Walters's avatar
Colin Walters committed
252
253
        return True

254
255
256
257
258
    def _adjust_container_type(self, parent, node, annotations):
        if ANN_ARRAY in annotations:
            self._apply_annotations_array(parent, node, annotations)
        elif ANN_ELEMENT_TYPE in annotations:
            self._apply_annotations_element_type(parent, node, annotations)
Colin Walters's avatar
Colin Walters committed
259

260
        if isinstance(node.type, ast.Array):
261
            self._check_array_element_type(node.type, annotations)
262

263
    def _resolve(self, type_str, type_node=None, node=None, parent=None):
Colin Walters's avatar
Colin Walters committed
264
265
266
267
        def grab_one(type_str, resolver, top_combiner, combiner):
            """Return a complete type, and the trailing string part after it.
            Use resolver() on each identifier, and combiner() on the parts of
            each complete type. (top_combiner is used on the top-most type.)"""
268
            bits = re.split(r'([,<>()])', type_str, 1)
269
            first, sep, rest = [bits[0], '', ''] if (len(bits) == 1) else bits
Colin Walters's avatar
Colin Walters committed
270
            args = [resolver(first)]
271
272
273
            if sep == '<' or sep == '(':
                lastsep = '>' if (sep == '<') else ')'
                while sep != lastsep:
Colin Walters's avatar
Colin Walters committed
274
275
276
277
278
279
                    next, rest = grab_one(rest, resolver, combiner, combiner)
                    args.append(next)
                    sep, rest = rest[0], rest[1:]
            else:
                rest = sep + rest
            return top_combiner(*args), rest
280

Colin Walters's avatar
Colin Walters committed
281
282
283
        def resolver(ident):
            res = self._transformer.create_type_from_user_string(ident)
            return res
284

Colin Walters's avatar
Colin Walters committed
285
286
287
288
289
290
291
        def combiner(base, *rest):
            if not rest:
                return base
            if isinstance(base, ast.List) and len(rest) == 1:
                return ast.List(base.name, *rest)
            if isinstance(base, ast.Map) and len(rest) == 2:
                return ast.Map(*rest)
292
293
            message.warn(
                "Too many parameters in type specification %r" % (type_str, ))
Colin Walters's avatar
Colin Walters committed
294
            return base
295

Colin Walters's avatar
Colin Walters committed
296
        def top_combiner(base, *rest):
297
298
            if type_node is not None and isinstance(type_node, ast.Type):
                base.is_const = type_node.is_const
Colin Walters's avatar
Colin Walters committed
299
300
301
302
            return combiner(base, *rest)

        result, rest = grab_one(type_str, resolver, top_combiner, combiner)
        if rest:
303
304
            message.warn("Trailing components in type specification %r" % (
                type_str, ))
305
306

        if not result.resolved:
307
            position = None
308
            if parent is not None and isinstance(parent, ast.Function):
309
                text = parent.symbol
310
                position = self._get_position(parent, node)
311
            else:
312
                text = type_str
313
            message.warn_node(parent, "%s: Unknown type: %r" %
314
                              (text, type_str), positions=position)
315
316
317
318
319
        return result

    def _resolve_toplevel(self, type_str, type_node=None, node=None, parent=None):
        """Like _resolve(), but attempt to preserve more attributes of original type."""
        result = self._resolve(type_str, type_node=type_node, node=node, parent=parent)
Johan Dahlin's avatar
Johan Dahlin committed
320
321
322
323
        # If we replace a node with a new type (such as an annotated) we
        # might lose the ctype from the original node.
        if type_node is not None:
            result.ctype = type_node.ctype
Colin Walters's avatar
Colin Walters committed
324
325
        return result

326
327
328
329
    def _get_position(self, func, param):
        block = self._blocks.get(func.symbol)
        if block:
            if isinstance(param, ast.Parameter):
330
                part = block.params.get(param.argname)
331
            elif isinstance(param, ast.Return):
332
                part = block.tags.get(TAG_RETURNS)
333
            else:
334
                part = None
335

336
337
            if part.position:
                return part.position
338
339
340

        return block.position

341
    def _check_array_element_type(self, array, annotations):
342
343
344
        array_type = array.array_type
        element_type = array.element_type

345
346
347
        # GPtrArrays are allowed to contain non basic types
        # (except enums and flags) or basic types that are
        # as big as a gpointer
348
349
350
351
        if array_type == ast.Array.GLIB_PTRARRAY:
            if ((element_type in ast.BASIC_GIR_TYPES and not element_type in ast.POINTER_TYPES)
            or isinstance(element_type, (ast.Enum, ast.Bitfield))):
                message.warn("invalid (element-type) for a GPtrArray, "
352
                             "must be a pointer", annotations.position)
353

354
        # GByteArrays have (element-type) guint8 by default
355
356
        if array_type == ast.Array.GLIB_BYTEARRAY:
            if element_type == ast.TYPE_ANY:
357
                array.element_type = ast.TYPE_UINT8
358
            elif not element_type in [ast.TYPE_UINT8, ast.TYPE_INT8, ast.TYPE_CHAR]:
359
360
                message.warn("invalid (element-type) for a GByteArray, "
                             "must be one of guint8, gint8 or gchar",
361
                             annotations.position)
362

363
    def _apply_annotations_array(self, parent, node, annotations):
364
365
366
        element_type_options = annotations.get(ANN_ELEMENT_TYPE)
        if element_type_options:
            element_type_node = self._resolve(element_type_options[0],
367
                                              node.type, node, parent)
Colin Walters's avatar
Colin Walters committed
368
369
370
371
372
373
        elif isinstance(node.type, ast.Array):
            element_type_node = node.type.element_type
        else:
            # We're assuming here that Foo* with an (array) annotation
            # and no (element-type) means array of Foo
            element_type_node = node.type.clone()
374
            # The element's ctype is the array's dereferenced
375
            if element_type_node.ctype is not None and element_type_node.ctype.endswith('*'):
376
                element_type_node.ctype = element_type_node.ctype[:-1]
Colin Walters's avatar
Colin Walters committed
377
378
379
380
381

        if isinstance(node.type, ast.Array):
            array_type = node.type.array_type
        else:
            array_type = None
382
383
384
385
386
387

        array_options = annotations.get(ANN_ARRAY)
        container_type = ast.Array(array_type, element_type_node, ctype=node.type.ctype,
                                   is_const=node.type.is_const)
        if OPT_ARRAY_ZERO_TERMINATED in array_options:
            container_type.zeroterminated = array_options.get(OPT_ARRAY_ZERO_TERMINATED) == '1'
388
389
        else:
            container_type.zeroterminated = False
390
391
392

        length = array_options.get(OPT_ARRAY_LENGTH)
        if length:
393
394
395
396
397
398
399
400
401
            if isinstance(parent, ast.Compound):
                paramname = self._get_validate_field_name(parent, length, node)
            else:
                paramname = self._get_validate_parameter_name(parent, length, node)
                if paramname:
                    param = parent.get_parameter(paramname)
                    param.direction = node.direction
                    if param.direction == ast.PARAM_DIRECTION_OUT:
                        param.transfer = ast.PARAM_TRANSFER_FULL
Colin Walters's avatar
Colin Walters committed
402
            if paramname:
403
                container_type.length_param_name = paramname
404
        fixed = array_options.get(OPT_ARRAY_FIXED_SIZE)
Colin Walters's avatar
Colin Walters committed
405
        if fixed:
Johan Dahlin's avatar
Johan Dahlin committed
406
407
            try:
                container_type.size = int(fixed)
408
            except (TypeError, ValueError):
Johan Dahlin's avatar
Johan Dahlin committed
409
410
                # Already warned in annotationparser.py
                return
Colin Walters's avatar
Colin Walters committed
411
412
        node.type = container_type

413
    def _apply_annotations_element_type(self, parent, node, annotations):
414
415
        element_type_options = annotations.get(ANN_ELEMENT_TYPE)
        if element_type_options is None:
416
417
            return

Colin Walters's avatar
Colin Walters committed
418
        if isinstance(node.type, ast.List):
419
            if len(element_type_options) != 1:
420
                message.warn(
421
                    '"element-type" annotation for a list must have exactly '
422
                    'one option, not %d options' % (len(element_type_options), ),
423
                    annotations.position)
424
                return
425
            node.type.element_type = self._resolve(element_type_options[0],
426
                                                   node.type, node, parent)
Colin Walters's avatar
Colin Walters committed
427
        elif isinstance(node.type, ast.Map):
428
            if len(element_type_options) != 2:
429
                message.warn(
430
                    '"element-type" annotation for a hash table must have exactly '
431
                    'two options, not %d option(s)' % (len(element_type_options), ),
432
                    annotations.position)
433
                return
434
            node.type.key_type = self._resolve(element_type_options[0],
435
                                               node.type, node, parent)
436
            node.type.value_type = self._resolve(element_type_options[1],
437
                                                 node.type, node, parent)
Colin Walters's avatar
Colin Walters committed
438
        elif isinstance(node.type, ast.Array):
439
            if len(element_type_options) != 1:
440
                message.warn(
441
                    '"element-type" annotation for an array must have exactly '
442
                    'one option, not %d options' % (len(element_type_options), ),
443
                    annotations.position)
444
                return
445
            node.type.element_type = self._resolve(element_type_options[0],
446
                                                   node.type, node, parent)
Colin Walters's avatar
Colin Walters committed
447
        else:
448
449
450
            message.warn(
                "Unknown container %r for element-type annotation" % (node.type, ),
                annotations.position)
Colin Walters's avatar
Colin Walters committed
451
452
453
454
455
456
457
458
459
460
461

    def _get_transfer_default_param(self, parent, node):
        if node.direction in [ast.PARAM_DIRECTION_INOUT,
                              ast.PARAM_DIRECTION_OUT]:
            if node.caller_allocates:
                return ast.PARAM_TRANSFER_NONE
            return ast.PARAM_TRANSFER_FULL
        return ast.PARAM_TRANSFER_NONE

    def _get_transfer_default_returntype_basic(self, typeval):
        if (typeval.is_equiv(ast.BASIC_GIR_TYPES)
462
463
        or typeval.is_const
        or typeval.is_equiv(ast.TYPE_NONE)):
Colin Walters's avatar
Colin Walters committed
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
            return ast.PARAM_TRANSFER_NONE
        elif typeval.is_equiv(ast.TYPE_STRING):
            # Non-const strings default to FULL
            return ast.PARAM_TRANSFER_FULL
        elif typeval.target_fundamental:
            # This looks like just GType right now
            return None
        return None

    def _is_gi_subclass(self, typeval, supercls_type):
        cls = self._transformer.lookup_typenode(typeval)
        assert cls, str(typeval)
        supercls = self._transformer.lookup_typenode(supercls_type)
        assert supercls
        if cls is supercls:
            return True
480
481
        if cls.parent_type and cls.parent_type.target_giname != 'GObject.Object':
            return self._is_gi_subclass(cls.parent_type, supercls_type)
Colin Walters's avatar
Colin Walters committed
482
483
484
485
486
487
488
489
490
491
492
493
        return False

    def _get_transfer_default_return(self, parent, node):
        typeval = node.type
        basic = self._get_transfer_default_returntype_basic(typeval)
        if basic:
            return basic
        if not typeval.target_giname:
            return None
        target = self._transformer.lookup_typenode(typeval)
        if isinstance(target, ast.Alias):
            return self._get_transfer_default_returntype_basic(target.target)
Colin Walters's avatar
Colin Walters committed
494
        elif (isinstance(target, ast.Boxed)
495
496
              or (isinstance(target, (ast.Record, ast.Union))
                  and (target.gtype_name is not None or target.foreign))):
Colin Walters's avatar
Colin Walters committed
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
            return ast.PARAM_TRANSFER_FULL
        elif isinstance(target, (ast.Enum, ast.Bitfield)):
            return ast.PARAM_TRANSFER_NONE
        # Handle constructors specially here
        elif isinstance(parent, ast.Function) and parent.is_constructor:
            if isinstance(target, ast.Class):
                initially_unowned_type = ast.Type(target_giname='GObject.InitiallyUnowned')
                initially_unowned = self._transformer.lookup_typenode(initially_unowned_type)
                if initially_unowned and self._is_gi_subclass(typeval, initially_unowned_type):
                    return ast.PARAM_TRANSFER_NONE
                else:
                    return ast.PARAM_TRANSFER_FULL
            elif isinstance(target, (ast.Record, ast.Union)):
                return ast.PARAM_TRANSFER_FULL
            else:
512
                raise AssertionError("Invalid constructor")
Colin Walters's avatar
Colin Walters committed
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
        elif isinstance(target, (ast.Class, ast.Record, ast.Union)):
            # Explicitly no default for these
            return None
        else:
            return None

    def _get_transfer_default(self, parent, node):
        if node.type.is_equiv(ast.TYPE_NONE) or isinstance(node.type, ast.Varargs):
            return ast.PARAM_TRANSFER_NONE
        elif isinstance(node, ast.Parameter):
            return self._get_transfer_default_param(parent, node)
        elif isinstance(node, ast.Return):
            return self._get_transfer_default_return(parent, node)
        elif isinstance(node, ast.Field):
            return ast.PARAM_TRANSFER_NONE
        elif isinstance(node, ast.Property):
            return ast.PARAM_TRANSFER_NONE
        else:
            raise AssertionError(node)

    def _apply_annotations_param_ret_common(self, parent, node, tag):
534
        annotations = tag.annotations if tag else {}
Colin Walters's avatar
Colin Walters committed
535

536
537
538
        type_annotation = annotations.get(ANN_TYPE)
        if type_annotation:
            node.type = self._resolve_toplevel(type_annotation[0],
539
                                               node.type, node, parent)
Colin Walters's avatar
Colin Walters committed
540
541
542

        caller_allocates = False
        annotated_direction = None
543
        if ANN_INOUT in annotations:
Colin Walters's avatar
Colin Walters committed
544
            annotated_direction = ast.PARAM_DIRECTION_INOUT
545
        elif ANN_OUT in annotations:
Colin Walters's avatar
Colin Walters committed
546
            annotated_direction = ast.PARAM_DIRECTION_OUT
547
548
549

            options = annotations[ANN_OUT]
            if len(options) == 0:
Colin Walters's avatar
Colin Walters committed
550
                if node.type.target_giname and node.type.ctype:
551
                    target = self._transformer.lookup_giname(node.type.target_giname)
552
                    target = self._transformer.resolve_aliases(target)
553
554
555
                    has_double_indirection = '**' in node.type.ctype
                    is_structure_or_union = isinstance(target, (ast.Record, ast.Union))
                    caller_allocates = (not has_double_indirection and is_structure_or_union)
Colin Walters's avatar
Colin Walters committed
556
557
                else:
                    caller_allocates = False
558
559
560
561
562
563
            else:
                option = options[0]
                if option == OPT_OUT_CALLER_ALLOCATES:
                    caller_allocates = True
                elif option == OPT_OUT_CALLEE_ALLOCATES:
                    caller_allocates = False
564
        elif ANN_IN in annotations:
Colin Walters's avatar
Colin Walters committed
565
566
567
568
569
570
571
572
            annotated_direction = ast.PARAM_DIRECTION_IN

        if (annotated_direction is not None) and (annotated_direction != node.direction):
            node.direction = annotated_direction
            node.caller_allocates = caller_allocates
            # Also reset the transfer default if we're toggling direction
            node.transfer = self._get_transfer_default(parent, node)

573
574
575
        transfer_annotation = annotations.get(ANN_TRANSFER)
        if transfer_annotation and len(transfer_annotation) == 1:
            transfer = transfer_annotation[0]
Johan Dahlin's avatar
Johan Dahlin committed
576
577
578
            if transfer == OPT_TRANSFER_FLOATING:
                transfer = OPT_TRANSFER_NONE
            node.transfer = transfer
Colin Walters's avatar
Colin Walters committed
579

580
        self._adjust_container_type(parent, node, annotations)
Colin Walters's avatar
Colin Walters committed
581

582
        if (ANN_ALLOW_NONE in annotations
583
584
        or node.type.target_giname == 'Gio.AsyncReadyCallback'
        or node.type.target_giname == 'Gio.Cancellable'):
Colin Walters's avatar
Colin Walters committed
585
586
            node.allow_none = True

587
        if tag and tag.description:
588
            node.doc = tag.description
Colin Walters's avatar
Colin Walters committed
589

590
        if ANN_SKIP in annotations:
591
592
            node.skip = True

593
        if annotations:
594
595
596
597
            attributes_annotation = annotations.get(ANN_ATTRIBUTES)
            if attributes_annotation is not None:
                for key, value in attributes_annotation.items():
                    if value:
598
                        node.attributes[key] = value
Colin Walters's avatar
Colin Walters committed
599
600
601
602
603

    def _apply_annotations_annotated(self, node, block):
        if block is None:
            return

604
605
        if block.description:
            node.doc = block.description
Colin Walters's avatar
Colin Walters committed
606

607
        since_tag = block.tags.get(TAG_SINCE)
Colin Walters's avatar
Colin Walters committed
608
        if since_tag is not None:
609
610
            if since_tag.value:
                node.version = since_tag.value
611
612
            if since_tag.description:
                node.version_doc = since_tag.description
Colin Walters's avatar
Colin Walters committed
613

614
        deprecated_tag = block.tags.get(TAG_DEPRECATED)
Colin Walters's avatar
Colin Walters committed
615
        if deprecated_tag is not None:
616
            if deprecated_tag.value:
617
                node.deprecated = deprecated_tag.value
618
            if deprecated_tag.description:
619
                node.deprecated_doc = deprecated_tag.description
Colin Walters's avatar
Colin Walters committed
620

621
        stability_tag = block.tags.get(TAG_STABILITY)
Evan Nemerson's avatar
Evan Nemerson committed
622
        if stability_tag is not None:
623
624
            if stability_tag.value:
                node.stability = stability_tag.value
625
626
            if stability_tag.description:
                node.stability_doc = stability_tag.description
Evan Nemerson's avatar
Evan Nemerson committed
627

628
629
630
631
        attributes_annotation = block.annotations.get(ANN_ATTRIBUTES)
        if attributes_annotation is not None:
            for key, value in attributes_annotation.items():
                if value:
632
                    node.attributes[key] = value
Colin Walters's avatar
Colin Walters committed
633

634
        if ANN_SKIP in block.annotations:
Colin Walters's avatar
Colin Walters committed
635
636
            node.skip = True

637
        if ANN_FOREIGN in block.annotations:
Colin Walters's avatar
Colin Walters committed
638
639
            node.foreign = True

640
        if ANN_CONSTRUCTOR in block.annotations and isinstance(node, ast.Function):
Tomeu Vizoso's avatar
Tomeu Vizoso committed
641
642
            node.is_constructor = True

643
        if ANN_METHOD in block.annotations:
Tomeu Vizoso's avatar
Tomeu Vizoso committed
644
645
            node.is_method = True

646
647
648
649
    def _apply_annotations_alias(self, node, chain):
        block = self._get_block(node)
        self._apply_annotations_annotated(node, block)

Colin Walters's avatar
Colin Walters committed
650
    def _apply_annotations_param(self, parent, param, tag):
651
652
        annotations = tag.annotations if tag else {}

653
        if isinstance(parent, (ast.Function, ast.VFunction)):
654
655
656
            scope_annotation = annotations.get(ANN_SCOPE)
            if scope_annotation and len(scope_annotation) == 1:
                param.scope = scope_annotation[0]
657
                param.transfer = ast.PARAM_TRANSFER_NONE
Colin Walters's avatar
Colin Walters committed
658

659
660
            destroy_annotation = annotations.get(ANN_DESTROY)
            if destroy_annotation:
Colin Walters's avatar
Colin Walters committed
661
                param.destroy_name = self._get_validate_parameter_name(parent,
662
                                                                       destroy_annotation[0],
Colin Walters's avatar
Colin Walters committed
663
664
665
666
667
668
669
670
                                                                       param)
                if param.destroy_name is not None:
                    param.scope = ast.PARAM_SCOPE_NOTIFIED
                    destroy_param = parent.get_parameter(param.destroy_name)
                    # This is technically bogus; we're setting the scope on the destroy
                    # itself.  But this helps avoid tripping a warning from finaltransformer,
                    # since we don't have a way right now to flag this callback a destroy.
                    destroy_param.scope = ast.PARAM_SCOPE_NOTIFIED
671
672
673

            closure_annotation = annotations.get(ANN_CLOSURE)
            if closure_annotation and len(closure_annotation) == 1:
Colin Walters's avatar
Colin Walters committed
674
                param.closure_name = self._get_validate_parameter_name(parent,
675
676
                                                                   closure_annotation[0],
                                                                   param)
Colin Walters's avatar
Colin Walters committed
677
        elif isinstance(parent, ast.Callback):
678
            if ANN_CLOSURE in annotations:
Colin Walters's avatar
Colin Walters committed
679
680
681
682
683
684
685
686
687
688
                # For callbacks, (closure) appears without an
                # argument, and tags a parameter that is a closure. We
                # represent it (weirdly) in the gir and typelib by
                # setting param.closure_name to itself.
                param.closure_name = param.argname

        self._apply_annotations_param_ret_common(parent, param, tag)

    def _apply_annotations_return(self, parent, return_, block):
        if block:
689
            tag = block.tags.get(TAG_RETURNS)
Colin Walters's avatar
Colin Walters committed
690
691
692
693
694
        else:
            tag = None
        self._apply_annotations_param_ret_common(parent, return_, tag)

    def _apply_annotations_params(self, parent, params, block):
695
        declparams = set([])
696
        if parent.instance_parameter:
697
698
699
700
701
            if block:
                doc_param = block.params.get(parent.instance_parameter.argname)
            else:
                doc_param = None
            self._apply_annotations_param(parent, parent.instance_parameter, doc_param)
702
            declparams.add(parent.instance_parameter.argname)
703

Colin Walters's avatar
Colin Walters committed
704
705
        for param in params:
            if block:
706
                doc_param = block.params.get(param.argname)
Colin Walters's avatar
Colin Walters committed
707
            else:
708
709
                doc_param = None
            self._apply_annotations_param(parent, param, doc_param)
710
            declparams.add(param.argname)
711
712
713

        if not block:
            return
714
715
716
717
718
719
720
        docparams = set(block.params)

        unknown = docparams - declparams
        unused = declparams - docparams

        for doc_name in unknown:
            if len(unused) == 0:
721
                text = ''
722
723
724
            elif len(unused) == 1:
                (param, ) = unused
                text = ', should be %r' % (param, )
725
            else:
726
                text = ', should be one of %s' % (', '.join(repr(p) for p in unused), )
727

728
            param = block.params.get(doc_name)
729
730
            message.warn('%s: unknown parameter %r in documentation '
                         'comment%s' % (block.name, doc_name, text),
731
                param.position)
Colin Walters's avatar
Colin Walters committed
732
733
734
735
736
737
738
739
740

    def _apply_annotations_callable(self, node, chain, block):
        self._apply_annotations_annotated(node, block)
        self._apply_annotations_params(node, node.parameters, block)
        self._apply_annotations_return(node, node.retval, block)

    def _apply_annotations_field(self, parent, block, field):
        if not block:
            return
741
        tag = block.params.get(field.name)
Colin Walters's avatar
Colin Walters committed
742
743
        if not tag:
            return
744
745
746
        type_annotation = tag.annotations.get(ANN_TYPE)
        if type_annotation:
            field.type = self._transformer.create_type_from_user_string(type_annotation[0])
747
        field.doc = tag.description
748
        try:
749
            self._adjust_container_type(parent, field, tag.annotations)
750
        except AttributeError as ex:
751
            print ex
Colin Walters's avatar
Colin Walters committed
752
753

    def _apply_annotations_property(self, parent, prop):
754
755
        prefix = self._get_annotation_name(parent)
        block = self._blocks.get('%s:%s' % (prefix, prop.name))
Colin Walters's avatar
Colin Walters committed
756
757
758
        self._apply_annotations_annotated(prop, block)
        if not block:
            return
759
760
761
        transfer_annotation = block.annotations.get(ANN_TRANSFER)
        if transfer_annotation is not None:
            transfer = transfer_annotation[0]
Johan Dahlin's avatar
Johan Dahlin committed
762
763
764
            if transfer == OPT_TRANSFER_FLOATING:
                transfer = OPT_TRANSFER_NONE
            prop.transfer = transfer
Colin Walters's avatar
Colin Walters committed
765
766
        else:
            prop.transfer = self._get_transfer_default(parent, prop)
767
768
769
        type_annotation = block.annotations.get(ANN_TYPE)
        if type_annotation:
            prop.type = self._resolve_toplevel(type_annotation[0], prop.type, prop, parent)
Colin Walters's avatar
Colin Walters committed
770
771

    def _apply_annotations_signal(self, parent, signal):
772
        names = []
773
774
        prefix = self._get_annotation_name(parent)
        block = self._blocks.get('%s::%s' % (prefix, signal.name))
775
776
777
778
779
780
781
782
783
784
785
786

        if block:
            self._apply_annotations_annotated(signal, block)

            # We're only attempting to name the signal parameters if
            # the number of parameters (@foo) is the same or greater
            # than the number of signal parameters
            if len(block.params) > len(signal.parameters):
                names = block.params.items()
                # Resolve real parameter names early, so that in later
                # phase we can refer to them while resolving annotations.
                for i, param in enumerate(signal.parameters):
787
                    param.argname, tag = names[i + 1]
788
789
790
            elif len(signal.parameters) != 0:
                # Only warn about missing params if there are actually parameters
                # besides implicit self.
791
792
793
                message.warn("incorrect number of parameters in comment block, "
                             "parameter annotations will be ignored.", block.position)

Colin Walters's avatar
Colin Walters committed
794
795
        for i, param in enumerate(signal.parameters):
            if names:
796
                name, tag = names[i + 1]
797
798
799
800
801
                if tag:
                    type_annotation = tag.annotations.get(ANN_TYPE)
                    if type_annotation:
                        param.type = self._resolve_toplevel(type_annotation[0], param.type,
                                                            param, parent)
Colin Walters's avatar
Colin Walters committed
802
803
804
805
806
            else:
                tag = None
            self._apply_annotations_param(signal, param, tag)
        self._apply_annotations_return(signal, signal.retval, block)

Johan Dahlin's avatar
Johan Dahlin committed
807
    def _apply_annotations_constant(self, node):
808
809
        block = self._get_block(node)
        if block is None:
Johan Dahlin's avatar
Johan Dahlin committed
810
            return
811
812
813

        self._apply_annotations_annotated(node, block)

814
815
816
        value_annotation = block.annotations.get(ANN_VALUE)
        if value_annotation:
            node.value = value_annotation[0]
Johan Dahlin's avatar
Johan Dahlin committed
817

818
819
820
821
822
    def _apply_annotations_enum_members(self, node, block):
        if block is None:
            return

        for m in node.members:
823
824
825
            param = block.params.get(m.symbol, None)
            if param and param.description:
                m.doc = param.description
826

Colin Walters's avatar
Colin Walters committed
827
828
    def _pass_read_annotations2(self, node, chain):
        if isinstance(node, ast.Function):
829
830
831
832
833
834
835
            block = self._blocks.get(node.symbol)

            self._apply_annotation_rename_to(node, chain, block)

            # Handle virtual invokers
            parent = chain[-1] if chain else None
            if (block and parent):
836
                virtual_annotation = block.annotations.get(ANN_VFUNC)
837
                if virtual_annotation:
838
                    invoker_name = virtual_annotation[0]
839
840
841
842
843
844
845
846
847
848
849
                    matched = False
                    for vfunc in parent.virtual_methods:
                        if vfunc.name == invoker_name:
                            matched = True
                            vfunc.invoker = node.name
                            # Also merge in annotations
                            self._apply_annotations_callable(vfunc, [parent], block)
                            break
                    if not matched:
                        message.warn_node(node,
                            "Virtual slot %r not found for %r annotation" % (invoker_name,
850
                                                                             ANN_VFUNC))
Colin Walters's avatar
Colin Walters committed
851
852
        return True

853
854
855
856
857
858
859
860
861
862
863
    def _resolve_and_filter_type_list(self, typelist):
        """Given a list of Type instances, return a new list of types with
the ones that failed to resolve removed."""
        # Create a copy we'll modify
        new_typelist = list(typelist)
        for typeval in typelist:
            resolved = self._transformer.resolve_type(typeval)
            if not resolved:
                new_typelist.remove(typeval)
        return new_typelist

Colin Walters's avatar
Colin Walters committed
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
    def _pass_type_resolution(self, node, chain):
        if isinstance(node, ast.Alias):
            self._transformer.resolve_type(node.target)
        if isinstance(node, ast.Callable):
            for parameter in node.parameters:
                self._transformer.resolve_type(parameter.type)
            self._transformer.resolve_type(node.retval.type)
        if isinstance(node, ast.Constant):
            self._transformer.resolve_type(node.value_type)
        if isinstance(node, (ast.Class, ast.Interface, ast.Record, ast.Union)):
            for field in node.fields:
                if field.anonymous_node:
                    pass
                else:
                    self._transformer.resolve_type(field.type)
        if isinstance(node, (ast.Class, ast.Interface)):
            for parent in node.parent_chain:
                try:
                    self._transformer.resolve_type(parent)
883
                except ValueError:
Colin Walters's avatar
Colin Walters committed
884
885
886
                    continue
                target = self._transformer.lookup_typenode(parent)
                if target:
887
                    node.parent_type = parent
Colin Walters's avatar
Colin Walters committed
888
                    break
889
            else:
890
                if isinstance(node, ast.Interface):
891
                    node.parent_type = ast.Type(target_giname='GObject.Object')
Colin Walters's avatar
Colin Walters committed
892
893
894
895
896
897
            for prop in node.properties:
                self._transformer.resolve_type(prop.type)
            for sig in node.signals:
                for param in sig.parameters:
                    self._transformer.resolve_type(param.type)
        if isinstance(node, ast.Class):
898
            node.interfaces = self._resolve_and_filter_type_list(node.interfaces)
Colin Walters's avatar
Colin Walters committed
899
        if isinstance(node, ast.Interface):
900
            node.prerequisites = self._resolve_and_filter_type_list(node.prerequisites)
Colin Walters's avatar
Colin Walters committed
901
902
        return True

Colin Walters's avatar
Colin Walters committed
903
    def _pair_quarks_with_enums(self):
Colin Walters's avatar
Colin Walters committed
904
905
906
907
908
909
910
911
        # self._uscore_type_names is an authoritative mapping of types
        # to underscored versions, since it is based on get_type() methods;
        # but only covers enums that are registered as GObject enums.
        # Create a fallback mapping based on all known enums in this module.
        uscore_enums = {}
        for enum in self._namespace.itervalues():
            if not isinstance(enum, ast.Enum):
                continue
912
            uscored = to_underscores_noprefix(enum.name).lower()
Colin Walters's avatar
Colin Walters committed
913
            uscore_enums[uscored] = enum
914
            uscore_enums[enum.name] = enum
Colin Walters's avatar
Colin Walters committed
915
916

        for node in self._namespace.itervalues():
917
            if not isinstance(node, ast.ErrorQuarkFunction):
Colin Walters's avatar
Colin Walters committed
918
                continue
919
920
921
922
            full = node.symbol[:-len('_quark')]
            ns, short = self._transformer.split_csymbol(node.symbol)
            short = short[:-len('_quark')]
            if full == "g_io_error":
Colin Walters's avatar
Colin Walters committed
923
924
925
926
927
928
                # Special case; GIOError was already taken forcing GIOErrorEnum
                assert self._namespace.name == 'Gio'
                enum = self._namespace.get('IOErrorEnum')
            else:
                enum = self._uscore_type_names.get(short)
                if enum is None:
929
                    enum = uscore_enums.get(short)
Colin Walters's avatar
Colin Walters committed
930
            if enum is not None:
931
                enum.error_domain = node.error_domain
Colin Walters's avatar
Colin Walters committed
932
            else:
933
934
                message.warn_node(node,
                    """%s: Couldn't find corresponding enumeration""" % (node.symbol, ))
Colin Walters's avatar
Colin Walters committed
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959

    def _split_uscored_by_type(self, uscored):
        """'uscored' should be an un-prefixed uscore string.  This
function searches through the namespace for the longest type which
prefixes uscored, and returns (type, suffix).  Example, assuming
namespace Gtk, type is TextBuffer:

_split_uscored_by_type(text_buffer_try_new) -> (ast.Class(TextBuffer), 'try_new')"""
        node = None
        count = 0
        prev_split_count = -1
        while True:
            components = uscored.rsplit('_', count)
            if len(components) == prev_split_count:
                return None
            prev_split_count = len(components)
            type_string = components[0]
            node = self._uscore_type_names.get(type_string)
            if node:
                return (node, '_'.join(components[1:]))
            count += 1

    def _pair_function(self, func):
        """Check to see whether a toplevel function should be a
method or constructor of some type."""
960
961
962

        # Ignore internal symbols and type metadata functions
        if func.symbol.startswith('_') or func.is_type_meta_function():
Colin Walters's avatar
Colin Walters committed
963
            return
964

Colin Walters's avatar
Colin Walters committed
965
966
        (ns, subsymbol) = self._transformer.split_csymbol(func.symbol)
        assert ns == self._namespace
967
        if self._is_constructor(func, subsymbol):
Tomeu Vizoso's avatar
Tomeu Vizoso committed
968
            self._set_up_constructor(func, subsymbol)
Colin Walters's avatar
Colin Walters committed
969
            return
970
        elif self._is_method(func, subsymbol):
Tomeu Vizoso's avatar
Tomeu Vizoso committed
971
            self._setup_method(func, subsymbol)
Colin Walters's avatar
Colin Walters committed
972
973
974
975
976
977
978
979
980
            return
        elif self._pair_static_method(func, subsymbol):
            return

    def _uscored_identifier_for_type(self, typeval):
        """Given a Type(target_giname='Foo.BarBaz'), return 'bar_baz'."""
        name = typeval.get_giname()
        return to_underscores_noprefix(name).lower()

Tomeu Vizoso's avatar
Tomeu Vizoso committed
981
    def _is_method(self, func, subsymbol):
Colin Walters's avatar
Colin Walters committed
982
        if not func.parameters:
983
984
985
            if func.is_method:
                message.warn_node(func,
                    '%s: Methods must have parameters' % (func.symbol, ))
Colin Walters's avatar