diff options
Diffstat (limited to 'giscanner')
89 files changed, 5581 insertions, 3866 deletions
diff --git a/giscanner/annotationmain.py b/giscanner/annotationmain.py index 4df6e831..618cf47f 100644 --- a/giscanner/annotationmain.py +++ b/giscanner/annotationmain.py @@ -21,11 +21,12 @@ import optparse from giscanner import message -from giscanner.annotationparser import AnnotationParser +from giscanner.annotationparser import GtkDocCommentBlockParser, GtkDocCommentBlockWriter from giscanner.scannermain import (get_preprocessor_option_group, create_source_scanner, process_packages) + def annotation_main(args): parser = optparse.OptionParser('%prog [options] sources') @@ -57,14 +58,15 @@ def annotation_main(args): ss = create_source_scanner(options, args) if options.extract: - ap = AnnotationParser() - blocks = ap.parse(ss.get_comments()) + parser = GtkDocCommentBlockParser() + writer = GtkDocCommentBlockWriter(indent=False) + blocks = parser.parse_comment_blocks(ss.get_comments()) print '/' + ('*' * 60) + '/' print '/* THIS FILE IS GENERATED DO NOT EDIT */' print '/' + ('*' * 60) + '/' print for block in sorted(blocks.values()): - print block.to_gtk_doc() + print writer.write(block) print print print '/' + ('*' * 60) + '/' diff --git a/giscanner/annotationparser.py b/giscanner/annotationparser.py index 2ac1b0eb..a00bac1d 100644 --- a/giscanner/annotationparser.py +++ b/giscanner/annotationparser.py @@ -1,7 +1,9 @@ +# -*- coding: utf-8 -*- # -*- Mode: Python -*- + # GObject-Introspection - a framework for introspecting GObject libraries # Copyright (C) 2008-2010 Johan Dahlin -# Copyright (C) 2012 Dieter Verfaillie <dieterv@optionexplicit.be> +# Copyright (C) 2012-2013 Dieter Verfaillie <dieterv@optionexplicit.be> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -20,912 +22,2112 @@ # -# AnnotationParser - extract annotations from gtk-doc comments +''' +GTK-Doc comment block format +---------------------------- + +A GTK-Doc comment block is built out of multiple parts. Each part can be further +divided into fields which are separated by a colon ("``:``") delimiter. + +Known parts and the fields they are constructed from look like the following +(optional fields are enclosed in square brackets):: + + ┌───────────────────────────────────────────────────────────┐ + │ /** │ ─▷ start token + ├────────────────────┬──────────────────────────────────────┤ + │ * identifier_name │ [: annotations] │ ─▷ identifier part + ├────────────────────┼─────────────────┬────────────────────┤ + │ * @parameter_name │ [: annotations] │ : description │ ─▷ parameter part + ├────────────────────┴─────────────────┴────────────────────┤ + │ * │ ─▷ comment block description + │ * comment_block_description │ + ├─────────────┬─────────────────┬───────────┬───────────────┤ + │ * tag_name │ [: annotations] │ [: value] │ : description │ ─▷ tag part + ├─────────────┴─────────────────┴───────────┴───────────────┤ + │ */ │ ─▷ end token + └───────────────────────────────────────────────────────────┘ + +There are two conditions that must be met before a comment block is recognized +as a GTK-Doc comment block: + +#. The comment block is opened with a GTK-Doc start token ("``/**``") +#. The first line following the start token contains a valid identifier part + +Once a GTK-Doc comment block has been identified as such and has been stripped +from its start and end tokens the remaining parts have to be written in a +specific order: + +#. There must be exactly 1 `identifier` part on the first line of the + comment block which consists of: + + * a required `identifier_name` field + * an optional `annotations` field + +#. Zero or more `parameter` parts, each consisting of: + + * a required `parameter_name` field + * an optional `annotations` field + * a required `description` field (can be the empty string) + +#. One optional `comment block description` part which must begin with at + least 1 empty line signaling the start of this part. + +#. Zero or more `tag` parts, each consisting of: + + * a required `tag_name` field + * an optional `annotations` field + * an optional `value` field + * a required `description` field (can be the empty string) +Additionally, the following restrictions are in effect: +#. Separating parts with an empty line: + + * `identifier` and `parameter` parts cannot be separated from each other by + an empty line as this would signal the start of the + `comment block description` part (see above). + * it is required to separate the `comment block description` part from the + `identifier` or `parameter` parts with an empty line (see above) + * `comment block description` and `tag` parts can optionally be separated + by an empty line + +#. Parts and fields cannot span multiple lines, except for: + + * the `comment_block_description` part + * `parameter description` and `tag description` fields + +#. Taking the above restrictions into account, spanning multiple paragraphs is + limited to the `comment block description` part and `tag description` fields. + +Refer to the `GTK-Doc manual`_ for more detailed usage information. + +.. _GTK-Doc manual: + http://developer.gnome.org/gtk-doc-manual/1.18/documenting.html.en +''' + + +from __future__ import absolute_import + +import os import re -from . import message -from .annotationpatterns import (COMMENT_START_RE, COMMENT_END_RE, - COMMENT_ASTERISK_RE, EMPTY_LINE_RE, - SECTION_RE, SYMBOL_RE, PROPERTY_RE, SIGNAL_RE, - PARAMETER_RE, DESCRIPTION_TAG_RE, TAG_RE, - MULTILINE_ANNOTATION_CONTINUATION_RE) -from .odict import odict +from collections import namedtuple +from operator import ne, gt, lt + +from .collections import Counter, OrderedDict +from .message import Position, warn, error # GTK-Doc comment block parts -PART_IDENTIFIER = 'identifier' -PART_PARAMETERS = 'parameters' -PART_DESCRIPTION = 'description' -PART_TAGS = 'tags' - -# Identifiers -IDENTIFIER_SECTION = 'section' -IDENTIFIER_SYMBOL = 'symbol' -IDENTIFIER_PROPERTY = 'property' -IDENTIFIER_SIGNAL = 'signal' - -# Tags - annotations applied to comment blocks -TAG_VFUNC = 'virtual' -TAG_SINCE = 'since' -TAG_STABILITY = 'stability' +PART_IDENTIFIER = 0 +PART_PARAMETERS = 1 +PART_DESCRIPTION = 2 +PART_TAGS = 3 + +# GTK-Doc comment block tags +# 1) Basic GTK-Doc tags. +# Note: This list cannot be extended unless the GTK-Doc project defines new tags. TAG_DEPRECATED = 'deprecated' TAG_RETURNS = 'returns' -TAG_RETURNVALUE = 'return value' +TAG_SINCE = 'since' +TAG_STABILITY = 'stability' + +GTKDOC_TAGS = [TAG_DEPRECATED, + TAG_RETURNS, + TAG_SINCE, + TAG_STABILITY] + +# 2) Deprecated basic GTK-Doc tags. +# Note: This list cannot be extended unless the GTK-Doc project defines new deprecated tags. +TAG_DESCRIPTION = 'description' +TAG_RETURN_VALUE = 'return value' + +DEPRECATED_GTKDOC_TAGS = [TAG_DESCRIPTION, + TAG_RETURN_VALUE] + +# 3) Deprecated GObject-Introspection tags. +# Unfortunately, these where accepted by old versions of this module. +TAG_RETURN = 'return' +TAG_RETURNS_VALUE = 'returns value' + +DEPRECATED_GI_TAGS = [TAG_RETURN, + TAG_RETURNS_VALUE] + +# 4) Deprecated GObject-Introspection annotation tags. +# Accepted by old versions of this module while they should have been +# annotations on the identifier part instead. +# Note: This list can not be extended ever again. The GObject-Introspection project is not +# allowed to invent GTK-Doc tags. Please create new annotations instead. TAG_ATTRIBUTES = 'attributes' -TAG_RENAME_TO = 'rename to' -TAG_TYPE = 'type' -TAG_UNREF_FUNC = 'unref func' +TAG_GET_VALUE_FUNC = 'get value func' TAG_REF_FUNC = 'ref func' +TAG_RENAME_TO = 'rename to' TAG_SET_VALUE_FUNC = 'set value func' -TAG_GET_VALUE_FUNC = 'get value func' TAG_TRANSFER = 'transfer' +TAG_TYPE = 'type' +TAG_UNREF_FUNC = 'unref func' TAG_VALUE = 'value' -_ALL_TAGS = [TAG_VFUNC, - TAG_SINCE, - TAG_STABILITY, - TAG_DEPRECATED, - TAG_RETURNS, - TAG_RETURNVALUE, - TAG_ATTRIBUTES, - TAG_RENAME_TO, - TAG_TYPE, - TAG_UNREF_FUNC, - TAG_REF_FUNC, - TAG_SET_VALUE_FUNC, - TAG_GET_VALUE_FUNC, - TAG_TRANSFER, - TAG_VALUE] - -# Options - annotations for parameters and return values -OPT_ALLOW_NONE = 'allow-none' -OPT_ARRAY = 'array' -OPT_ATTRIBUTE = 'attribute' -OPT_CLOSURE = 'closure' -OPT_DESTROY = 'destroy' -OPT_ELEMENT_TYPE = 'element-type' -OPT_FOREIGN = 'foreign' -OPT_IN = 'in' -OPT_INOUT = 'inout' -OPT_INOUT_ALT = 'in-out' -OPT_OUT = 'out' -OPT_SCOPE = 'scope' -OPT_TRANSFER = 'transfer' -OPT_TYPE = 'type' -OPT_SKIP = 'skip' -OPT_CONSTRUCTOR = 'constructor' -OPT_METHOD = 'method' - -ALL_OPTIONS = [ - OPT_ALLOW_NONE, - OPT_ARRAY, - OPT_ATTRIBUTE, - OPT_CLOSURE, - OPT_DESTROY, - OPT_ELEMENT_TYPE, - OPT_FOREIGN, - OPT_IN, - OPT_INOUT, - OPT_INOUT_ALT, - OPT_OUT, - OPT_SCOPE, - OPT_TRANSFER, - OPT_TYPE, - OPT_SKIP, - OPT_CONSTRUCTOR, - OPT_METHOD] - -# Array options - array specific annotations +TAG_VFUNC = 'virtual' + +DEPRECATED_GI_ANN_TAGS = [TAG_ATTRIBUTES, + TAG_GET_VALUE_FUNC, + TAG_REF_FUNC, + TAG_RENAME_TO, + TAG_SET_VALUE_FUNC, + TAG_TRANSFER, + TAG_TYPE, + TAG_UNREF_FUNC, + TAG_VALUE, + TAG_VFUNC] + +ALL_TAGS = GTKDOC_TAGS + DEPRECATED_GTKDOC_TAGS + DEPRECATED_GI_TAGS + DEPRECATED_GI_ANN_TAGS + +# GObject-Introspection annotation start/end tokens +ANN_LPAR = '(' +ANN_RPAR = ')' + +# GObject-Introspection annotations +# 1) Supported annotations +# Note: when adding new annotations, GTK-Doc project's gtkdoc-mkdb needs to be modified too! +ANN_ALLOW_NONE = 'allow-none' +ANN_ARRAY = 'array' +ANN_ATTRIBUTES = 'attributes' +ANN_CLOSURE = 'closure' +ANN_CONSTRUCTOR = 'constructor' +ANN_DESTROY = 'destroy' +ANN_ELEMENT_TYPE = 'element-type' +ANN_FOREIGN = 'foreign' +ANN_GET_VALUE_FUNC = 'get-value-func' +ANN_IN = 'in' +ANN_INOUT = 'inout' +ANN_METHOD = 'method' +ANN_OUT = 'out' +ANN_REF_FUNC = 'ref-func' +ANN_RENAME_TO = 'rename-to' +ANN_SCOPE = 'scope' +ANN_SET_VALUE_FUNC = 'set-value-func' +ANN_SKIP = 'skip' +ANN_TRANSFER = 'transfer' +ANN_TYPE = 'type' +ANN_UNREF_FUNC = 'unref-func' +ANN_VFUNC = 'virtual' +ANN_VALUE = 'value' + +GI_ANNS = [ANN_ALLOW_NONE, + ANN_ARRAY, + ANN_ATTRIBUTES, + ANN_CLOSURE, + ANN_CONSTRUCTOR, + ANN_DESTROY, + ANN_ELEMENT_TYPE, + ANN_FOREIGN, + ANN_GET_VALUE_FUNC, + ANN_IN, + ANN_INOUT, + ANN_METHOD, + ANN_OUT, + ANN_REF_FUNC, + ANN_RENAME_TO, + ANN_SCOPE, + ANN_SET_VALUE_FUNC, + ANN_SKIP, + ANN_TRANSFER, + ANN_TYPE, + ANN_UNREF_FUNC, + ANN_VFUNC, + ANN_VALUE] + +# 2) Deprecated GObject-Introspection annotations +ANN_ATTRIBUTE = 'attribute' +ANN_INOUT_ALT = 'in-out' + +DEPRECATED_GI_ANNS = [ANN_ATTRIBUTE, + ANN_INOUT_ALT] + +ALL_ANNOTATIONS = GI_ANNS + DEPRECATED_GI_ANNS +DICT_ANNOTATIONS = [ANN_ARRAY, ANN_ATTRIBUTES] +LIST_ANNOTATIONS = [ann for ann in ALL_ANNOTATIONS if ann not in DICT_ANNOTATIONS] + +# (array) annotation options OPT_ARRAY_FIXED_SIZE = 'fixed-size' OPT_ARRAY_LENGTH = 'length' OPT_ARRAY_ZERO_TERMINATED = 'zero-terminated' -# Out options -OPT_OUT_CALLER_ALLOCATES = 'caller-allocates' +ARRAY_OPTIONS = [OPT_ARRAY_FIXED_SIZE, + OPT_ARRAY_LENGTH, + OPT_ARRAY_ZERO_TERMINATED] + +# (out) annotation options OPT_OUT_CALLEE_ALLOCATES = 'callee-allocates' +OPT_OUT_CALLER_ALLOCATES = 'caller-allocates' + +OUT_OPTIONS = [OPT_OUT_CALLEE_ALLOCATES, + OPT_OUT_CALLER_ALLOCATES] -# Scope options +# (scope) annotation options OPT_SCOPE_ASYNC = 'async' OPT_SCOPE_CALL = 'call' OPT_SCOPE_NOTIFIED = 'notified' -# Transfer options -OPT_TRANSFER_NONE = 'none' +SCOPE_OPTIONS = [OPT_SCOPE_ASYNC, + OPT_SCOPE_CALL, + OPT_SCOPE_NOTIFIED] + +# (transfer) annotation options OPT_TRANSFER_CONTAINER = 'container' -OPT_TRANSFER_FULL = 'full' OPT_TRANSFER_FLOATING = 'floating' +OPT_TRANSFER_FULL = 'full' +OPT_TRANSFER_NONE = 'none' +TRANSFER_OPTIONS = [OPT_TRANSFER_CONTAINER, + OPT_TRANSFER_FLOATING, + OPT_TRANSFER_FULL, + OPT_TRANSFER_NONE] + + +# Pattern used to normalize different types of line endings +LINE_BREAK_RE = re.compile(r'\r\n|\r|\n', re.UNICODE) + +# Pattern matching the start token of a comment block. +COMMENT_BLOCK_START_RE = re.compile( + r''' + ^ # start + (?P<code>.*?) # whitespace, code, ... + \s* # 0 or more whitespace characters + (?P<token>/\*{2}(?![\*/])) # 1 forward slash character followed + # by exactly 2 asterisk characters + # and not followed by a slash character + \s* # 0 or more whitespace characters + (?P<comment>.*?) # GTK-Doc comment text + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching the end token of a comment block. +COMMENT_BLOCK_END_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<comment>.*?) # GTK-Doc comment text + \s* # 0 or more whitespace characters + (?P<token>\*+/) # 1 or more asterisk characters followed + # by exactly 1 forward slash character + (?P<code>.*?) # whitespace, code, ... + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching the ' * ' at the beginning of every +# line inside a comment block. +COMMENT_ASTERISK_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<comment>.*?) # invalid comment text + \s* # 0 or more whitespace characters + \* # 1 asterisk character + \s? # 0 or 1 whitespace characters + # WARNING: removing more than 1 + # whitespace character breaks + # embedded example program indentation + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching the indentation level of a line (used +# to get the indentation before and after the ' * '). +INDENTATION_RE = re.compile( + r''' + ^ + (?P<indentation>\s*) # 0 or more whitespace characters + .* + $ + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching an empty line. +EMPTY_LINE_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching SECTION identifiers. +SECTION_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + SECTION # SECTION + \s* # 0 or more whitespace characters + (?P<delimiter>:?) # delimiter + \s* # 0 or more whitespace characters + (?P<section_name>\w\S+?) # section name + \s* # 0 or more whitespace characters + :? # invalid delimiter + \s* # 0 or more whitespace characters + $ + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching symbol (function, constant, struct and enum) identifiers. +SYMBOL_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<symbol_name>[\w-]*\w) # symbol name + \s* # 0 or more whitespace characters + (?P<delimiter>:?) # delimiter + \s* # 0 or more whitespace characters + (?P<fields>.*?) # annotations + description + \s* # 0 or more whitespace characters + :? # invalid delimiter + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching property identifiers. +PROPERTY_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<class_name>[\w]+) # class name + \s* # 0 or more whitespace characters + :{1} # 1 required colon + \s* # 0 or more whitespace characters + (?P<property_name>[\w-]*\w) # property name + \s* # 0 or more whitespace characters + (?P<delimiter>:?) # delimiter + \s* # 0 or more whitespace characters + (?P<fields>.*?) # annotations + description + \s* # 0 or more whitespace characters + :? # invalid delimiter + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching signal identifiers. +SIGNAL_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<class_name>[\w]+) # class name + \s* # 0 or more whitespace characters + :{2} # 2 required colons + \s* # 0 or more whitespace characters + (?P<signal_name>[\w-]*\w) # signal name + \s* # 0 or more whitespace characters + (?P<delimiter>:?) # delimiter + \s* # 0 or more whitespace characters + (?P<fields>.*?) # annotations + description + \s* # 0 or more whitespace characters + :? # invalid delimiter + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching parameters. +PARAMETER_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + @ # @ character + (?P<parameter_name>[\w-]*\w|.*?\.\.\.) # parameter name + \s* # 0 or more whitespace characters + :{1} # 1 required delimiter + \s* # 0 or more whitespace characters + (?P<fields>.*?) # annotations + description + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching tags. +_all_tags = '|'.join(ALL_TAGS).replace(' ', r'\s') +TAG_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<tag_name>''' + _all_tags + r''') # tag name + \s* # 0 or more whitespace characters + :{1} # 1 required delimiter + \s* # 0 or more whitespace characters + (?P<fields>.*?) # annotations + value + description + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE | re.IGNORECASE) + +# Pattern matching value and description fields for TAG_DEPRECATED & TAG_SINCE tags. +TAG_VALUE_VERSION_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<value>([0-9\.])*) # value + \s* # 0 or more whitespace characters + (?P<delimiter>:?) # delimiter + \s* # 0 or more whitespace characters + (?P<description>.*?) # description + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE) + +# Pattern matching value and description fields for TAG_STABILITY tags. +TAG_VALUE_STABILITY_RE = re.compile( + r''' + ^ # start + \s* # 0 or more whitespace characters + (?P<value>(stable|unstable|private|internal)?) # value + \s* # 0 or more whitespace characters + (?P<delimiter>:?) # delimiter + \s* # 0 or more whitespace characters + (?P<description>.*?) # description + \s* # 0 or more whitespace characters + $ # end + ''', + re.UNICODE | re.VERBOSE | re.IGNORECASE) + + +class GtkDocAnnotations(OrderedDict): + ''' + An ordered dictionary mapping annotation names to annotation options (if any). Annotation + options can be either a :class:`list`, a :class:`giscanner.collections.OrderedDict` + (depending on the annotation name)or :const:`None`. + ''' + + __slots__ = ('position') + + def __init__(self, position=None): + OrderedDict.__init__(self) + + #: A :class:`giscanner.message.Position` instance specifying the location of the + #: annotations in the source file or :const:`None`. + self.position = position -class DocBlock(object): - def __init__(self, name): - self.name = name - self.options = DocOptions() - self.value = None - self.tags = odict() - self.comment = None - self.params = odict() - self.position = None +class GtkDocAnnotatable(object): + ''' + Base class for GTK-Doc comment block parts that can be annotated. + ''' - def __cmp__(self, other): - return cmp(self.name, other.name) + __slots__ = ('position', 'annotations') - def __repr__(self): - return '<DocBlock %r %r>' % (self.name, self.options) + #: A :class:`tuple` of annotation name constants that are valid for this object. Annotation + #: names not in this :class:`tuple` will be reported as *unknown* by :func:`validate`. The + #: :attr:`valid_annotations` class attribute should be overridden by subclasses. + valid_annotations = () - def set_position(self, position): + def __init__(self, position=None): + #: A :class:`giscanner.message.Position` instance specifying the location of the + #: annotatable comment block part in the source file or :const:`None`. self.position = position - self.options.position = position - - def get_tag(self, name): - return self.tags.get(name) - - def get_param(self, name): - return self.params.get(name) - - def to_gtk_doc(self): - options = '' - if self.options: - options += ' ' - options += ' '.join('(%s)' % o for o in self.options) - lines = [self.name] - if 'SECTION' not in self.name: - lines[0] += ':' - lines[0] += options - for param in self.params.values(): - lines.append(param.to_gtk_doc_param()) - lines.append('') - for l in self.comment.split('\n'): - lines.append(l) - if self.tags: - lines.append('') - for tag in self.tags.values(): - lines.append(tag.to_gtk_doc_tag()) - - comment = '' - comment += '/**\n' - for line in lines: - line = line.rstrip() - if line: - comment += ' * %s\n' % (line, ) - else: - comment += ' *\n' - comment += ' */\n' - return comment + + #: A :class:`GtkDocAnnotations` instance representing the annotations + #: applied to this :class:`GtkDocAnnotatable` instance. + self.annotations = GtkDocAnnotations() + + def __repr__(self): + return '<GtkDocAnnotatable %r %r>' % (self.annotations, ) def validate(self): - for param in self.params.values(): - param.validate() + ''' + Validate annotations stored by the :class:`GtkDocAnnotatable` instance, if any. + ''' - for tag in self.tags.values(): - tag.validate() + if self.annotations: + position = self.annotations.position + + for ann_name, options in self.annotations.items(): + if ann_name in self.valid_annotations: + validate = getattr(self, '_do_validate_' + ann_name.replace('-', '_')) + validate(position, ann_name, options) + elif ann_name in ALL_ANNOTATIONS: + # Not error() as ann_name might be valid in some newer + # GObject-Instrospection version. + warn('unexpected annotation: %s' % (ann_name, ), position) + else: + # Not error() as ann_name might be valid in some newer + # GObject-Instrospection version. + warn('unknown annotation: %s' % (ann_name, ), position) + def _validate_options(self, position, ann_name, n_options, expected_n_options, operator, + message): + ''' + Validate the number of options held by an annotation according to the test + ``operator(n_options, expected_n_options)``. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param n_options: number of options held by the annotation + :param expected_n_options: number of expected options + :param operator: an operator function from python's :mod:`operator` module, for example + :func:`operator.ne` or :func:`operator.lt` + :param message: warning message used when the test + ``operator(n_options, expected_n_options)`` fails. + ''' -class DocTag(object): + if n_options == 0: + t = 'none' + else: + t = '%d' % (n_options, ) - def __init__(self, block, name): - self.block = block - self.name = name - self.options = DocOptions() - self.comment = None - self.value = '' - self.position = None + if expected_n_options == 0: + s = 'no options' + elif expected_n_options == 1: + s = 'one option' + else: + s = '%d options' % (expected_n_options, ) - def __repr__(self): - return '<DocTag %r %r>' % (self.name, self.options) + if operator(n_options, expected_n_options): + warn('"%s" annotation %s %s, %s given' % (ann_name, message, s, t), position) - def _validate_option(self, name, value, required=False, - n_params=None, choices=None): - if required and value is None: - message.warn('%s annotation needs a value' % ( - name, ), self.position) - return + def _validate_annotation(self, position, ann_name, options, choices=None, + exact_n_options=None, min_n_options=None, max_n_options=None): + ''' + Validate an annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to be validated + :param choices: an iterable of allowed option names or :const:`None` to skip this test + :param exact_n_options: exact number of expected options or :const:`None` to skip this test + :param min_n_options: minimum number of expected options or :const:`None` to skip this test + :param max_n_options: maximum number of expected options or :const:`None` to skip this test + ''' - if n_params is not None: - if n_params == 0: - s = 'no value' - elif n_params == 1: - s = 'one value' - else: - s = '%d values' % (n_params, ) - if ((n_params > 0 and (value is None or value.length() != n_params)) or - n_params == 0 and value is not None): - if value is None: - length = 0 - else: - length = value.length() - message.warn('%s annotation needs %s, not %d' % ( - name, s, length), self.position) - return - - if choices is not None: - valuestr = value.one() - if valuestr not in choices: - message.warn('invalid %s annotation value: %r' % ( - name, valuestr, ), self.position) - return - - def _validate_array(self, option, value): - if value is None: + n_options = len(options) + + if exact_n_options is not None: + self._validate_options(position, + ann_name, n_options, exact_n_options, ne, 'needs') + + if min_n_options is not None: + self._validate_options(position, + ann_name, n_options, min_n_options, lt, 'takes at least') + + if max_n_options is not None: + self._validate_options(position, + ann_name, n_options, max_n_options, gt, 'takes at most') + + if options and choices is not None: + option = options[0] + if option not in choices: + warn('invalid "%s" annotation option: "%s"' % (ann_name, option), position) + + def _do_validate_allow_none(self, position, ann_name, options): + ''' + Validate the ``(allow-none)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options held by the annotation + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=0) + + def _do_validate_array(self, position, ann_name, options): + ''' + Validate the ``(array)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options held by the annotation + ''' + + if len(options) == 0: return - for name, v in value.all().iteritems(): - if name in [OPT_ARRAY_ZERO_TERMINATED, OPT_ARRAY_FIXED_SIZE]: + for option, value in options.items(): + if option in [OPT_ARRAY_ZERO_TERMINATED, OPT_ARRAY_FIXED_SIZE]: try: - int(v) + int(value) except (TypeError, ValueError): - if v is None: - message.warn( - 'array option %s needs a value' % ( - name, ), - positions=self.position) + if value is None: + warn('"%s" annotation option "%s" needs a value' % (ann_name, option), + position) else: - message.warn( - 'invalid array %s option value %r, ' - 'must be an integer' % (name, v, ), - positions=self.position) - elif name == OPT_ARRAY_LENGTH: - if v is None: - message.warn( - 'array option length needs a value', - positions=self.position) + warn('invalid "%s" annotation option "%s" value "%s", must be an integer' % + (ann_name, option, value), + position) + elif option == OPT_ARRAY_LENGTH: + if value is None: + warn('"%s" annotation option "length" needs a value' % (ann_name, ), + position) else: - message.warn( - 'invalid array annotation value: %r' % ( - name, ), self.position) - - def _validate_closure(self, option, value): - if value is not None and value.length() > 1: - message.warn( - 'closure takes at most 1 value, %d given' % ( - value.length()), self.position) - - def _validate_element_type(self, option, value): - self._validate_option(option, value, required=True) - if value is None: - message.warn( - 'element-type takes at least one value, none given', - self.position) - return - if value.length() > 2: - message.warn( - 'element-type takes at most 2 values, %d given' % ( - value.length()), self.position) - return + warn('invalid "%s" annotation option: "%s"' % (ann_name, option), + position) - def _validate_out(self, option, value): - if value is None: - return - if value.length() > 1: - message.warn( - 'out annotation takes at most 1 value, %d given' % ( - value.length()), self.position) - return - value_str = value.one() - if value_str not in [OPT_OUT_CALLEE_ALLOCATES, - OPT_OUT_CALLER_ALLOCATES]: - message.warn("out annotation value is invalid: %r" % ( - value_str), self.position) - return + def _do_validate_attributes(self, position, ann_name, options): + ''' + Validate the ``(attributes)`` annotation. - def set_position(self, position): - self.position = position - self.options.position = position - - def _get_gtk_doc_value(self): - def serialize_one(option, value, fmt, fmt2): - if value: - if type(value) != str: - value = ' '.join((serialize_one(k, v, '%s=%s', '%s') - for k, v in value.all().iteritems())) - return fmt % (option, value) - else: - return fmt2 % (option, ) - annotations = [] - for option, value in self.options.iteritems(): - annotations.append( - serialize_one(option, value, '(%s %s)', '(%s)')) - if annotations: - return ' '.join(annotations) + ': ' - else: - return self.value + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' - def to_gtk_doc_param(self): - return '@%s: %s%s' % (self.name, self._get_gtk_doc_value(), self.comment) + # The 'attributes' annotation allows free form annotations. + pass - def to_gtk_doc_tag(self): - return '%s: %s%s' % (self.name.capitalize(), - self._get_gtk_doc_value(), - self.comment or '') + def _do_validate_closure(self, position, ann_name, options): + ''' + Validate the ``(closure)`` annotation. - def validate(self): - if self.name == TAG_ATTRIBUTES: - # The 'Attributes:' tag allows free form annotations so the - # validation below is most certainly going to fail. - return + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' - for option in self.options: - value = self.options[option] - if option == OPT_ALLOW_NONE: - self._validate_option(option, value, n_params=0) - elif option == OPT_ARRAY: - self._validate_array(option, value) - elif option == OPT_ATTRIBUTE: - self._validate_option(option, value, n_params=2) - elif option == OPT_CLOSURE: - self._validate_closure(option, value) - elif option == OPT_DESTROY: - self._validate_option(option, value, n_params=1) - elif option == OPT_ELEMENT_TYPE: - self._validate_element_type(option, value) - elif option == OPT_FOREIGN: - self._validate_option(option, value, n_params=0) - elif option == OPT_IN: - self._validate_option(option, value, n_params=0) - elif option in [OPT_INOUT, OPT_INOUT_ALT]: - self._validate_option(option, value, n_params=0) - elif option == OPT_OUT: - self._validate_out(option, value) - elif option == OPT_SCOPE: - self._validate_option( - option, value, required=True, - n_params=1, - choices=[OPT_SCOPE_ASYNC, - OPT_SCOPE_CALL, - OPT_SCOPE_NOTIFIED]) - elif option == OPT_SKIP: - self._validate_option(option, value, n_params=0) - elif option == OPT_TRANSFER: - self._validate_option( - option, value, required=True, - n_params=1, - choices=[OPT_TRANSFER_FULL, - OPT_TRANSFER_CONTAINER, - OPT_TRANSFER_NONE, - OPT_TRANSFER_FLOATING]) - elif option == OPT_TYPE: - self._validate_option(option, value, required=True, - n_params=1) - elif option == OPT_CONSTRUCTOR: - self._validate_option(option, value, n_params=0) - elif option == OPT_METHOD: - self._validate_option(option, value, n_params=0) - else: - message.warn('invalid annotation option: %s' % (option, ), - self.position) + self._validate_annotation(position, ann_name, options, max_n_options=1) + + def _do_validate_constructor(self, position, ann_name, options): + ''' + Validate the ``(constructor)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=0) + + def _do_validate_destroy(self, position, ann_name, options): + ''' + Validate the ``(destroy)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_element_type(self, position, ann_name, options): + ''' + Validate the ``(element)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, min_n_options=1, max_n_options=2) + + def _do_validate_foreign(self, position, ann_name, options): + ''' + Validate the ``(foreign)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=0) + + def _do_validate_get_value_func(self, position, ann_name, options): + ''' + Validate the ``(value-func)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_in(self, position, ann_name, options): + ''' + Validate the ``(in)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=0) + + def _do_validate_inout(self, position, ann_name, options): + ''' + Validate the ``(in-out)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=0) + + def _do_validate_method(self, position, ann_name, options): + ''' + Validate the ``(method)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=0) + + def _do_validate_out(self, position, ann_name, options): + ''' + Validate the ``(out)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, max_n_options=1, + choices=OUT_OPTIONS) + + def _do_validate_ref_func(self, position, ann_name, options): + ''' + Validate the ``(ref-func)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_rename_to(self, position, ann_name, options): + ''' + Validate the ``(rename-to)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_scope(self, position, ann_name, options): + ''' + Validate the ``(scope)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1, + choices=SCOPE_OPTIONS) + + def _do_validate_set_value_func(self, position, ann_name, options): + ''' + Validate the ``(value-func)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_skip(self, position, ann_name, options): + ''' + Validate the ``(skip)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=0) + + def _do_validate_transfer(self, position, ann_name, options): + ''' + Validate the ``(transfer)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1, + choices=TRANSFER_OPTIONS) + + def _do_validate_type(self, position, ann_name, options): + ''' + Validate the ``(type)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_unref_func(self, position, ann_name, options): + ''' + Validate the ``(unref-func)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_value(self, position, ann_name, options): + ''' + Validate the ``(value)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) + + def _do_validate_virtual(self, position, ann_name, options): + ''' + Validate the ``(virtual)`` annotation. + + :param position: :class:`giscanner.message.Position` of the line in the source file + containing the annotation to be validated + :param ann_name: name of the annotation holding the options to validate + :param options: annotation options to validate + ''' + + self._validate_annotation(position, ann_name, options, exact_n_options=1) -class DocOptions(object): - def __init__(self): - self.values = [] +class GtkDocParameter(GtkDocAnnotatable): + ''' + Represents a GTK-Doc parameter part. + ''' + + __slots__ = ('name', 'description') + + valid_annotations = (ANN_ALLOW_NONE, ANN_ARRAY, ANN_ATTRIBUTES, ANN_CLOSURE, ANN_DESTROY, + ANN_ELEMENT_TYPE, ANN_IN, ANN_INOUT, ANN_OUT, ANN_SCOPE, ANN_SKIP, + ANN_TRANSFER, ANN_TYPE) + + def __init__(self, name, position=None): + GtkDocAnnotatable.__init__(self, position) + + #: Parameter name. + self.name = name + + #: Parameter description or :const:`None`. + self.description = None def __repr__(self): - return '<DocOptions %r>' % (self.values, ) + return '<GtkDocParameter %r %r>' % (self.name, self.annotations) + - def __getitem__(self, item): - for key, value in self.values: - if key == item: - return value - raise KeyError +class GtkDocTag(GtkDocAnnotatable): + ''' + Represents a GTK-Doc tag part. + ''' - def __nonzero__(self): - return bool(self.values) + __slots__ = ('name', 'value', 'description') - def __iter__(self): - return (k for k, v in self.values) + valid_annotations = (ANN_ALLOW_NONE, ANN_ARRAY, ANN_ATTRIBUTES, ANN_ELEMENT_TYPE, ANN_SKIP, + ANN_TRANSFER, ANN_TYPE) - def add(self, name, value): - self.values.append((name, value)) + def __init__(self, name, position=None): + GtkDocAnnotatable.__init__(self, position) - def get(self, item, default=None): - for key, value in self.values: - if key == item: - return value - return default + #: Tag name. + self.name = name - def getall(self, item): - for key, value in self.values: - if key == item: - yield value + #: Tag value or :const:`None`. + self.value = None - def iteritems(self): - return iter(self.values) + #: Tag description or :const:`None`. + self.description = None + def __repr__(self): + return '<GtkDocTag %r %r>' % (self.name, self.annotations) -class DocOption(object): - def __init__(self, tag, option): - self.tag = tag - self._array = [] - self._dict = {} - # (annotation option1=value1 option2=value2) etc - for p in option.split(' '): - if '=' in p: - name, value = p.split('=', 1) - else: - name = p - value = None - self._dict[name] = value - if value is None: - self._array.append(name) - else: - self._array.append((name, value)) +class GtkDocCommentBlock(GtkDocAnnotatable): + ''' + Represents a GTK-Doc comment block. + ''' + + __slots__ = ('code_before', 'code_after', 'indentation', + 'name', 'params', 'description', 'tags') + + #: Valid annotation names for the GTK-Doc comment block identifier part. + valid_annotations = (ANN_ATTRIBUTES, ANN_CONSTRUCTOR, ANN_FOREIGN, ANN_GET_VALUE_FUNC, + ANN_METHOD, ANN_REF_FUNC, ANN_RENAME_TO, ANN_SET_VALUE_FUNC, + ANN_SKIP, ANN_TRANSFER, ANN_TYPE, ANN_UNREF_FUNC, ANN_VALUE, ANN_VFUNC) + + def __init__(self, name, position=None): + GtkDocAnnotatable.__init__(self, position) + + #: Code preceding the GTK-Doc comment block start token ("``/**``"), if any. + self.code_before = None + + #: Code following the GTK-Doc comment block end token ("``*/``"), if any. + self.code_after = None + + #: List of indentation levels (preceding the "``*``") for all lines in the comment + #: block's source text. + self.indentation = [] + + #: Identifier name. + self.name = name + + #: Ordered dictionary mapping parameter names to :class:`GtkDocParameter` instances + #: applied to this :class:`GtkDocCommentBlock`. + self.params = OrderedDict() + + #: The GTK-Doc comment block description part. + self.description = None + + #: Ordered dictionary mapping tag names to :class:`GtkDocTag` instances + #: applied to this :class:`GtkDocCommentBlock`. + self.tags = OrderedDict() + + def __cmp__(self, other): + # Note: This is used by g-ir-annotation-tool, which does a ``sorted(blocks.values())``, + # meaning that keeping this around makes update-glib-annotations.py patches + # easier to review. + return cmp(self.name, other.name) def __repr__(self): - return '<DocOption %r>' % (self._array, ) - - def length(self): - return len(self._array) - - def one(self): - assert len(self._array) == 1 - return self._array[0] - - def flat(self): - return self._array - - def all(self): - return self._dict - - -class AnnotationParser(object): - """ - GTK-Doc comment block parser. - - Parses GTK-Doc comment blocks into a parse tree built out of :class:`DockBlock`, - :class:`DocTag`, :class:`DocOptions` and :class:`DocOption` objects. This - parser tries to accept malformed input whenever possible and does not emit - syntax errors. However, it does emit warnings at the slightest indication - of malformed input when possible. - - A GTK-Doc comment block can be constructed out of multiple parts that can - be combined to write different types of documentation. - See `GTK-Doc's documentation`_ to learn more about possible valid combinations. - Each part can be further divided into fields which are separated by `:` characters. - - Possible parts and the fields they are constructed from look like the - following (optional fields are enclosed in square brackets): - - .. code-block:: c - /** - * identifier_name: [annotations] - * @parameter_name: [annotations:] [description] - * - * description - * tag_name: [annotations:] [description] - */ - - - Parts and fields cannot span multiple lines, except for parameter descriptions, - tag descriptions and comment block descriptions. - - There has to be exactly 1 `identifier` part on the first line of the - comment block which consists of: - * an `identifier_name` field - * an optional `annotations` field - - There can be 0 or more `parameter` parts following the `identifier` part, - each consisting of: - * a `parameter_name` filed - * an optional `annotations` field - * an optional `description` field - - An empty lines signals the end of the `parameter` parts and the beginning - of the (free form) comment block `description` part. - - There can be 0 or 1 `description` parts following the `description` part. - - There can be 0 or more `tag` parts following the `description` part, - each consisting of: - * a `tag_name` field - * an optional `annotations` field - * an optional `description` field - - .. NOTE:: :class:`AnnotationParser` functionality is heavily based on gtkdoc-mkdb's - `ScanSourceFile()`_ function and is currently in sync with gtk-doc - commit `b41641b`_. - - .. _types of documentation: - http://developer.gnome.org/gtk-doc-manual/1.18/documenting.html.en + return '<GtkDocCommentBlock %r %r>' % (self.name, self.annotations) + + def validate(self): + ''' + Validate annotations applied to the :class:`GtkDocCommentBlock` identifier, parameters + and tags. + ''' + GtkDocAnnotatable.validate(self) + + for param in self.params.values(): + param.validate() + + for tag in self.tags.values(): + tag.validate() + + +#: Result object returned by :class:`GtkDocCommentBlockParser`._parse_annotations() +_ParseAnnotationsResult = namedtuple('Result', ['success', 'annotations', 'start_pos', 'end_pos']) + +#: Result object returned by :class:`GtkDocCommentBlockParser`._parse_fields() +_ParseFieldsResult = namedtuple('Result', ['success', 'annotations', 'description']) + + +class GtkDocCommentBlockParser(object): + ''' + Parse GTK-Doc comment blocks into a parse tree built out of :class:`GtkDocCommentBlock`, + :class:`GtkDocParameter`, :class:`GtkDocTag` and :class:`GtkDocAnnotations` + objects. This parser tries to accept malformed input whenever possible and does + not cause the process to exit on syntax errors. It does however emit: + + * warning messages at the slightest indication of recoverable malformed input and + * error messages for unrecoverable malformed input + + whenever possible. Recoverable, in this context, means that we can serialize the + :class:`GtkDocCommentBlock` instance using a :class:`GtkDocCommentBlockWriter` without + information being lost. It is usually a good idea to heed these warning and error messages + as malformed input can result in both: + + * invalid GTK-Doc output (HTML, pdf, ...) when the comment blocks are parsed + with GTK-Doc's gtkdoc-mkdb + * unexpected introspection behavior, for example missing parameters in the + generated .gir and .typelib files + + .. NOTE:: :class:`GtkDocCommentBlockParser` functionality is heavily based on gtkdoc-mkdb's + `ScanSourceFile()`_ function and is currently in sync with GTK-Doc + commit `47abcd5`_. + .. _ScanSourceFile(): - http://git.gnome.org/browse/gtk-doc/tree/gtkdoc-mkdb.in#n3722 - .. _b41641b: b41641bd75f870afff7561ceed8a08456da57565 - """ + http://git.gnome.org/browse/gtk-doc/tree/gtkdoc-mkdb.in#n3722 + .. _47abcd5: + https://git.gnome.org/browse/gtk-doc/commit/?id=47abcd53b8489ebceec9e394676512a181c1f1f6 + ''' - def parse(self, comments): - """ - Parses multiple GTK-Doc comment blocks. + def parse_comment_blocks(self, comments): + ''' + Parse multiple GTK-Doc comment blocks. - :param comments: a list of (comment, filename, lineno) tuples - :returns: a list of :class:`DocBlock` or ``None`` objects - """ + :param comments: an iterable of ``(comment, filename, lineno)`` tuples + :returns: a dictionary mapping identifier names to :class:`GtkDocCommentBlock` objects + ''' comment_blocks = {} - for comment in comments: - comment_block = self.parse_comment_block(comment) + for (comment, filename, lineno) in comments: + try: + comment_block = self.parse_comment_block(comment, filename, lineno) + except Exception: + error('unrecoverable parse error, please file a GObject-Introspection bug' + 'report including the complete comment block at the indicated location.', + Position(filename, lineno)) + continue if comment_block is not None: + # Note: previous versions of this parser did not check if an identifier was + # already stored in comment_blocks, so when different comment blocks where + # encountered documenting the same identifier the last comment block seen + # "wins". Keep this behavior for backwards compatibility, but emit a warning. if comment_block.name in comment_blocks: - message.warn("multiple comment blocks documenting '%s:' identifier." % - (comment_block.name), - comment_block.position) + firstseen = comment_blocks[comment_block.name] + path = os.path.dirname(firstseen.position.filename) + warn('multiple comment blocks documenting \'%s:\' identifier ' + '(already seen at %s).' % + (comment_block.name, firstseen.position.format(path)), + comment_block.position) - # Always store the block even if it's a duplicate for - # backward compatibility... comment_blocks[comment_block.name] = comment_block return comment_blocks - def parse_comment_block(self, comment): - """ - Parses a single GTK-Doc comment block. + def parse_comment_block(self, comment, filename, lineno): + ''' + Parse a single GTK-Doc comment block. - :param comment: a (comment, filename, lineno) tuple - :returns: a :class:`DocBlock` object or ``None`` - """ + :param comment: string representing the GTK-Doc comment block including it's + start ("``/**``") and end ("``*/``") tokens. + :param filename: source file name where the comment block originated from + :param lineno: line number in the source file where the comment block starts + :returns: a :class:`GtkDocCommentBlock` object or ``None`` + ''' - comment, filename, lineno = comment - comment_lines = list(enumerate(comment.split('\n'))) + code_before = '' + code_after = '' + comment_block_pos = Position(filename, lineno) + comment_lines = re.sub(LINE_BREAK_RE, '\n', comment).split('\n') + comment_lines_len = len(comment_lines) - # Check for the start the comment block. - if COMMENT_START_RE.search(comment_lines[0][1]): - del comment_lines[0] + # Check for the start of the comment block. + result = COMMENT_BLOCK_START_RE.match(comment_lines[0]) + if result: + # Skip single line comment blocks + if comment_lines_len == 1: + position = Position(filename, lineno) + marker = ' ' * result.end('code') + '^' + error('Skipping invalid GTK-Doc comment block:' + '\n%s\n%s' % (comment_lines[0], marker), + position) + return None + + code_before = result.group('code') + comment = result.group('comment') + + if code_before: + position = Position(filename, lineno) + marker = ' ' * result.end('code') + '^' + warn('GTK-Doc comment block start token "/**" should ' + 'not be preceded by code:\n%s\n%s' % (comment_lines[0], marker), + position) + + if comment: + position = Position(filename, lineno) + marker = ' ' * result.start('comment') + '^' + warn('GTK-Doc comment block start token "/**" should ' + 'not be followed by comment text:\n%s\n%s' % (comment_lines[0], marker), + position) + + comment_lines[0] = comment + else: + del comment_lines[0] else: # Not a GTK-Doc comment block. return None - # Check for the end the comment block. - if COMMENT_END_RE.search(comment_lines[-1][1]): - del comment_lines[-1] - - # If we get this far, we are inside a GTK-Doc comment block. - return self._parse_comment_block(comment_lines, filename, lineno) - - def _parse_comment_block(self, comment_lines, filename, lineno): - """ - Parses a single GTK-Doc comment block stripped from it's - comment start (/**) and comment end (*/) marker lines. - - :param comment_lines: GTK-Doc comment block stripped from it's comment - start (/**) and comment end (*/) marker lines - :param filename: source file name where the comment block originated from - :param lineno: line in the source file where the comment block starts - :returns: a :class:`DocBlock` object or ``None`` - - .. NOTE:: If you are tempted to refactor this method and split it - further up (for example into _parse_identifier(), _parse_parameters(), - _parse_description(), _parse_tags() methods) then please resist the - urge. It is considered important that this method should be more or - less easily comparable with gtkdoc-mkdb's `ScanSourceFile()`_ function. - - The different parsing steps are marked with a comment surrounded - by `#` characters in an attempt to make it clear what is going on. + # Check for the end of the comment block. + result = COMMENT_BLOCK_END_RE.match(comment_lines[-1]) + if result: + code_after = result.group('code') + comment = result.group('comment') + if code_after: + position = Position(filename, lineno + comment_lines_len - 1) + marker = ' ' * result.end('code') + '^' + warn('GTK-Doc comment block end token "*/" should ' + 'not be followed by code:\n%s\n%s' % (comment_lines[-1], marker), + position) + + if comment: + position = Position(filename, lineno + comment_lines_len - 1) + marker = ' ' * result.end('comment') + '^' + warn('GTK-Doc comment block end token "*/" should ' + 'not be preceded by comment text:\n%s\n%s' % (comment_lines[-1], marker), + position) + + comment_lines[-1] = comment + else: + del comment_lines[-1] + else: + # Not a GTK-Doc comment block. + return None - .. _ScanSourceFile(): - http://git.gnome.org/browse/gtk-doc/tree/gtkdoc-mkdb.in#n3722 - """ + # If we get this far, we must be inside something + # that looks like a GTK-Doc comment block. comment_block = None + identifier_warned = False + block_indent = [] + line_indent = None + part_indent = None in_part = None - identifier = None - current_param = None - current_tag = None + current_part = None returns_seen = False - for line_offset, line in comment_lines: - position = message.Position(filename, line_offset + lineno) + for line in comment_lines: + lineno += 1 + position = Position(filename, lineno) # Store the original line (without \n) and column offset # so we can generate meaningful warnings later on. original_line = line column_offset = 0 - # Get rid of ' * ' at start of the line. + # Store indentation level of the comment (before the ' * ') + result = INDENTATION_RE.match(line) + block_indent.append(result.group('indentation')) + + # Get rid of the ' * ' at the start of the line. result = COMMENT_ASTERISK_RE.match(line) if result: + comment = result.group('comment') + if comment: + marker = ' ' * result.start('comment') + '^' + error('invalid comment text:\n%s\n%s' % + (original_line, marker), + position) + column_offset = result.end(0) line = line[result.end(0):] + # Store indentation level of the line (after the ' * '). + result = INDENTATION_RE.match(line) + line_indent = len(result.group('indentation').replace('\t', ' ')) + #################################################################### # Check for GTK-Doc comment block identifier. #################################################################### - if not comment_block: - # The correct identifier name would have the colon at the end - # but maintransformer.py does not expect us to do that. So - # make sure to compute an identifier_name without the colon and - # a real_identifier_name with the colon. - - if not identifier: - result = SECTION_RE.search(line) - if result: - identifier = IDENTIFIER_SECTION - real_identifier_name = 'SECTION:%s' % (result.group('section_name')) - identifier_name = real_identifier_name - column = result.start('section_name') + column_offset - - if not identifier: - result = SYMBOL_RE.search(line) - if result: - identifier = IDENTIFIER_SYMBOL - real_identifier_name = '%s:' % (result.group('symbol_name')) - identifier_name = '%s' % (result.group('symbol_name')) - column = result.start('symbol_name') + column_offset + if comment_block is None: + result = SECTION_RE.match(line) + + if result: + identifier_name = 'SECTION:%s' % (result.group('section_name'), ) + identifier_delimiter = None + identifier_fields = None + identifier_fields_start = None + else: + result = PROPERTY_RE.match(line) - if not identifier: - result = PROPERTY_RE.search(line) if result: - identifier = IDENTIFIER_PROPERTY - real_identifier_name = '%s:%s:' % (result.group('class_name'), - result.group('property_name')) identifier_name = '%s:%s' % (result.group('class_name'), result.group('property_name')) - column = result.start('property_name') + column_offset - - if not identifier: - result = SIGNAL_RE.search(line) - if result: - identifier = IDENTIFIER_SIGNAL - real_identifier_name = '%s::%s:' % (result.group('class_name'), - result.group('signal_name')) - identifier_name = '%s::%s' % (result.group('class_name'), - result.group('signal_name')) - column = result.start('signal_name') + column_offset - - if identifier: - in_part = PART_IDENTIFIER - - comment_block = DocBlock(identifier_name) - comment_block.set_position(position) + identifier_delimiter = result.group('delimiter') + identifier_fields = result.group('fields') + identifier_fields_start = result.start('fields') + else: + result = SIGNAL_RE.match(line) + + if result: + identifier_name = '%s::%s' % (result.group('class_name'), + result.group('signal_name')) + identifier_delimiter = result.group('delimiter') + identifier_fields = result.group('fields') + identifier_fields_start = result.start('fields') + else: + result = SYMBOL_RE.match(line) - if 'colon' in result.groupdict() and result.group('colon') != ':': - colon_start = result.start('colon') - colon_column = column_offset + colon_start - marker = ' '*colon_column + '^' - message.warn("missing ':' at column %s:\n%s\n%s" % - (colon_column + 1, original_line, marker), - position) + if result: + identifier_name = '%s' % (result.group('symbol_name'), ) + identifier_delimiter = result.group('delimiter') + identifier_fields = result.group('fields') + identifier_fields_start = result.start('fields') - if 'annotations' in result.groupdict(): - comment_block.options = self.parse_options(comment_block, - result.group('annotations')) + if result: + in_part = PART_IDENTIFIER + part_indent = line_indent + + comment_block = GtkDocCommentBlock(identifier_name, comment_block_pos) + comment_block.code_before = code_before + comment_block.code_after = code_after + + if identifier_fields: + res = self._parse_annotations(position, + column_offset + identifier_fields_start, + original_line, + identifier_fields) + + if res.success: + if identifier_fields[res.end_pos:].strip(): + # Not an identifier due to invalid trailing description field + result = None + in_part = None + part_indent = None + comment_block = None + else: + comment_block.annotations = res.annotations + + if not identifier_delimiter and res.annotations: + marker_position = column_offset + result.start('delimiter') + marker = ' ' * marker_position + '^' + warn('missing ":" at column %s:\n%s\n%s' % + (marker_position + 1, original_line, marker), + position) - continue - else: - # If we get here, the identifier was not recognized, so - # ignore the rest of the block just like the old annotation - # parser did. Doing this is a bit more strict than - # gtkdoc-mkdb (which continues to search for the identifier - # until either it is found or the end of the block is - # reached). In practice, however, ignoring the block is the - # right thing to do because sooner or later some long - # descriptions will contain something matching an identifier - # pattern by accident. - marker = ' '*column_offset + '^' - message.warn('ignoring unrecognized GTK-Doc comment block, identifier not ' - 'found:\n%s\n%s' % (original_line, marker), - position) - - return None + if not result: + # Emit a single warning when the identifier is not found on the first line + if not identifier_warned: + identifier_warned = True + marker = ' ' * column_offset + '^' + error('identifier not found on the first line:\n%s\n%s' % + (original_line, marker), + position) + continue #################################################################### # Check for comment block parameters. #################################################################### - result = PARAMETER_RE.search(line) + result = PARAMETER_RE.match(line) if result: + part_indent = line_indent param_name = result.group('parameter_name') - param_annotations = result.group('annotations') - param_description = result.group('description') - - if in_part == PART_IDENTIFIER: - in_part = PART_PARAMETERS - - if in_part != PART_PARAMETERS: - column = result.start('parameter_name') + column_offset - marker = ' '*column + '^' - message.warn("'@%s' parameter unexpected at this location:\n%s\n%s" % - (param_name, original_line, marker), - position) - - # Old style GTK-Doc allowed return values to be specified as - # parameters instead of tags. - if param_name.lower() == TAG_RETURNS: + param_name_lower = param_name.lower() + param_fields = result.group('fields') + param_fields_start = result.start('fields') + marker = ' ' * (result.start('parameter_name') + column_offset) + '^' + + if in_part not in [PART_IDENTIFIER, PART_PARAMETERS]: + warn('"@%s" parameter unexpected at this location:\n%s\n%s' % + (param_name, original_line, marker), + position) + + in_part = PART_PARAMETERS + + if param_name_lower == TAG_RETURNS: + # Deprecated return value as parameter instead of tag param_name = TAG_RETURNS if not returns_seen: returns_seen = True else: - message.warn("encountered multiple 'Returns' parameters or tags for " - "'%s'." % (comment_block.name), - position) - elif param_name in comment_block.params.keys(): - column = result.start('parameter_name') + column_offset - marker = ' '*column + '^' - message.warn("multiple '@%s' parameters for identifier '%s':\n%s\n%s" % - (param_name, comment_block.name, original_line, marker), - position) - - tag = DocTag(comment_block, param_name) - tag.set_position(position) - tag.comment = param_description - if param_annotations: - tag.options = self.parse_options(tag, param_annotations) - if param_name == TAG_RETURNS: - comment_block.tags[param_name] = tag - else: - comment_block.params[param_name] = tag - current_param = tag + error('encountered multiple "Returns" parameters or tags for "%s".' % + (comment_block.name, ), + position) + + tag = GtkDocTag(TAG_RETURNS, position) + + if param_fields: + result = self._parse_fields(position, + column_offset + param_fields_start, + original_line, + param_fields) + if result.success: + tag.annotations = result.annotations + tag.description = result.description + comment_block.tags[TAG_RETURNS] = tag + current_part = tag + continue + elif (param_name == 'Varargs' + or (param_name.endswith('...') and param_name != '...')): + # Deprecated @Varargs notation or named __VA_ARGS__ instead of @... + warn('"@%s" parameter is deprecated, please use "@..." instead:\n%s\n%s' % + (param_name, original_line, marker), + position) + param_name = '...' + + if param_name in comment_block.params.keys(): + error('multiple "@%s" parameters for identifier "%s":\n%s\n%s' % + (param_name, comment_block.name, original_line, marker), + position) + + parameter = GtkDocParameter(param_name, position) + + if param_fields: + result = self._parse_fields(position, + column_offset + param_fields_start, + original_line, + param_fields) + if result.success: + parameter.annotations = result.annotations + parameter.description = result.description + + comment_block.params[param_name] = parameter + current_part = parameter continue #################################################################### # Check for comment block description. # - # When we are parsing comment block parameters or the comment block - # identifier (when there are no parameters) and encounter an empty - # line, we must be parsing the comment block description + # When we are parsing parameter parts or the identifier part (when + # there are no parameters) and encounter an empty line, we must be + # parsing the comment block description. + # + # Note: it is unclear why GTK-Doc does not allow paragraph breaks + # at this location as those might be handy describing + # parameters from time to time... #################################################################### - if (EMPTY_LINE_RE.search(line) - and (in_part == PART_IDENTIFIER or in_part == PART_PARAMETERS)): + if (EMPTY_LINE_RE.match(line) and in_part in [PART_IDENTIFIER, PART_PARAMETERS]): in_part = PART_DESCRIPTION + part_indent = line_indent continue #################################################################### # Check for GTK-Doc comment block tags. #################################################################### - result = TAG_RE.search(line) - if result: + result = TAG_RE.match(line) + if result and line_indent <= part_indent: + part_indent = line_indent tag_name = result.group('tag_name') - tag_annotations = result.group('annotations') - tag_description = result.group('description') + tag_name_lower = tag_name.lower() + tag_fields = result.group('fields') + tag_fields_start = result.start('fields') + marker = ' ' * (result.start('tag_name') + column_offset) + '^' + + if tag_name_lower in DEPRECATED_GI_ANN_TAGS: + # Deprecated GObject-Introspection specific tags. + # Emit a warning and transform these into annotations on the identifier + # instead, as agreed upon in http://bugzilla.gnome.org/show_bug.cgi?id=676133 + warn('GObject-Introspection specific GTK-Doc tag "%s" ' + 'has been deprecated, please use annotations on the identifier ' + 'instead:\n%s\n%s' % (tag_name, original_line, marker), + position) + + # Translate deprecated tag name into corresponding annotation name + ann_name = tag_name_lower.replace(' ', '-') + + if tag_name_lower == TAG_ATTRIBUTES: + transformed = '' + result = self._parse_fields(position, + result.start('tag_name') + column_offset, + line, + tag_fields.strip(), + False, + False) + + if result.success: + for annotation in result.annotations: + ann_options = self._parse_annotation_options_list(position, marker, + line, annotation) + n_options = len(ann_options) + if n_options == 1: + transformed = '%s %s' % (transformed, ann_options[0], ) + elif n_options == 2: + transformed = '%s %s=%s' % (transformed, ann_options[0], + ann_options[1]) + else: + # Malformed Attributes: tag + error('malformed "Attributes:" tag will be ignored:\n%s\n%s' % + (original_line, marker), + position) + transformed = None + + if transformed: + transformed = '%s %s' % (ann_name, transformed.strip()) + ann_name, docannotation = self._parse_annotation( + position, + column_offset + tag_fields_start, + original_line, + transformed) + stored_annotation = comment_block.annotations.get('attributes') + if stored_annotation: + error('Duplicate "Attributes:" annotation will ' + 'be ignored:\n%s\n%s' % (original_line, marker), + position) + else: + comment_block.annotations[ann_name] = docannotation + else: + ann_name, options = self._parse_annotation(position, + column_offset + tag_fields_start, + line, + '%s %s' % (ann_name, tag_fields)) + comment_block.annotations[ann_name] = options + + continue + elif tag_name_lower == TAG_DESCRIPTION: + # Deprecated GTK-Doc Description: tag + warn('GTK-Doc tag "Description:" has been deprecated:\n%s\n%s' % + (original_line, marker), + position) + + in_part = PART_DESCRIPTION - if in_part == PART_DESCRIPTION: + if comment_block.description is None: + comment_block.description = tag_fields + else: + comment_block.description += '\n%s' % (tag_fields, ) + continue + + # Now that the deprecated stuff is out of the way, continue parsing real tags + if (in_part == PART_DESCRIPTION + or (in_part == PART_PARAMETERS and not comment_block.description) + or (in_part == PART_IDENTIFIER and not comment_block.params and not + comment_block.description)): in_part = PART_TAGS if in_part != PART_TAGS: - column = result.start('tag_name') + column_offset - marker = ' '*column + '^' - message.warn("'%s:' tag unexpected at this location:\n%s\n%s" % - (tag_name, original_line, marker), - position) + in_part = PART_TAGS + warn('"%s:" tag unexpected at this location:\n%s\n%s' % + (tag_name, original_line, marker), + position) - if tag_name.lower() in [TAG_RETURNS, TAG_RETURNVALUE]: + if tag_name_lower in [TAG_RETURN, TAG_RETURNS, + TAG_RETURN_VALUE, TAG_RETURNS_VALUE]: if not returns_seen: returns_seen = True else: - message.warn("encountered multiple 'Returns' parameters or tags for " - "'%s'." % (comment_block.name), - position) - - tag = DocTag(comment_block, TAG_RETURNS) - tag.position = position - tag.comment = tag_description - if tag_annotations: - tag.options = self.parse_options(tag, tag_annotations) + error('encountered multiple return value parameters or tags for "%s".' % + (comment_block.name, ), + position) + + tag = GtkDocTag(TAG_RETURNS, position) + + if tag_fields: + result = self._parse_fields(position, + column_offset + tag_fields_start, + original_line, + tag_fields) + if result.success: + tag.annotations = result.annotations + tag.description = result.description + comment_block.tags[TAG_RETURNS] = tag - current_tag = tag + current_part = tag continue else: - if tag_name.lower() in comment_block.tags.keys(): - column = result.start('tag_name') + column_offset - marker = ' '*column + '^' - message.warn("multiple '%s:' tags for identifier '%s':\n%s\n%s" % - (tag_name, comment_block.name, original_line, marker), - position) - - tag = DocTag(comment_block, tag_name.lower()) - tag.position = position - tag.value = tag_description - if tag_annotations: - if tag_name.lower() == TAG_ATTRIBUTES: - tag.options = self.parse_options(tag, tag_annotations) - else: - message.warn("annotations not supported for tag '%s:'." % - (tag_name), - position) - comment_block.tags[tag_name.lower()] = tag - current_tag = tag + if tag_name_lower in comment_block.tags.keys(): + error('multiple "%s:" tags for identifier "%s":\n%s\n%s' % + (tag_name, comment_block.name, original_line, marker), + position) + + tag = GtkDocTag(tag_name_lower, position) + + if tag_fields: + result = self._parse_fields(position, + column_offset + tag_fields_start, + original_line, + tag_fields) + if result.success: + if result.annotations: + error('annotations not supported for tag "%s:".' % (tag_name, ), + position) + + if tag_name_lower in [TAG_DEPRECATED, TAG_SINCE]: + result = TAG_VALUE_VERSION_RE.match(result.description) + tag.value = result.group('value') + tag.description = result.group('description') + elif tag_name_lower == TAG_STABILITY: + result = TAG_VALUE_STABILITY_RE.match(result.description) + tag.value = result.group('value').capitalize() + tag.description = result.group('description') + + comment_block.tags[tag_name_lower] = tag + current_part = tag continue #################################################################### # If we get here, we must be in the middle of a multiline # comment block, parameter or tag description. #################################################################### - if in_part == PART_DESCRIPTION or in_part == PART_IDENTIFIER: - if not comment_block.comment: - # Backwards compatibility with old style GTK-Doc - # comment blocks where Description used to be a comment - # block tag. Simply get rid of 'Description:'. - line = re.sub(DESCRIPTION_TAG_RE, '', line) - comment_block.comment = line + if EMPTY_LINE_RE.match(line) is None: + line = line.rstrip() + + if in_part in [PART_IDENTIFIER, PART_DESCRIPTION]: + if not comment_block.description: + if in_part == PART_IDENTIFIER: + self._validate_multiline_annotation_continuation(line, original_line, + column_offset, position) + if comment_block.description is None: + comment_block.description = line else: - comment_block.comment += '\n' + line + comment_block.description += '\n' + line continue - elif in_part == PART_PARAMETERS: - self._validate_multiline_annotation_continuation(line, original_line, - column_offset, position) - - # Append to parameter description. - current_param.comment += ' ' + line.strip() - elif in_part == PART_TAGS: - self._validate_multiline_annotation_continuation(line, original_line, - column_offset, position) - - # Append to tag description. - if current_tag.name.lower() in [TAG_RETURNS, TAG_RETURNVALUE]: - current_tag.comment += ' ' + line.strip() + elif in_part in [PART_PARAMETERS, PART_TAGS]: + if not current_part.description: + self._validate_multiline_annotation_continuation(line, original_line, + column_offset, position) + if current_part.description is None: + current_part.description = line else: - current_tag.value += ' ' + line.strip() + current_part.description += '\n' + line + continue ######################################################################## # Finished parsing this comment block. ######################################################################## - # We have picked up a couple of \n characters that where not - # intended. Strip those. - if comment_block.comment: - comment_block.comment = comment_block.comment.strip() + if comment_block: + # We have picked up a couple of \n characters that where not + # intended. Strip those. + if comment_block.description: + comment_block.description = comment_block.description.strip() + + for tag in comment_block.tags.values(): + self._clean_description_field(tag) + + for param in comment_block.params.values(): + self._clean_description_field(param) + + comment_block.indentation = block_indent + comment_block.validate() + return comment_block else: - comment_block.comment = '' + return None + + def _clean_description_field(self, part): + ''' + Remove extraneous leading and trailing whitespace from description fields. + + :param part: a GTK-Doc comment block part having a description field + ''' + + if part.description: + if part.description.strip() == '': + part.description = None + else: + if EMPTY_LINE_RE.match(part.description.split('\n', 1)[0]): + part.description = part.description.rstrip() + else: + part.description = part.description.strip() + + def _validate_multiline_annotation_continuation(self, line, original_line, + column_offset, position): + ''' + Validate annotatable parts' source text ensuring annotations don't span multiple lines. + For example, the following comment block would result in a warning being emitted for + the forth line:: + + /** + * shiny_function: + * @array_: (out caller-allocates) (array) + * (element-type utf8) (transfer full): A beautiful array + */ + + :param line: line to validate, stripped from ("``*/``") at start of the line. + :param original_line: original line (including ("``*/``")) being validated + :param column_offset: number of characters stripped from `line` when ("``*/``") + was removed + :param position: :class:`giscanner.message.Position` of `line` in the source file + ''' + + result = self._parse_annotations(position, column_offset, original_line, line) + + if result.success and result.annotations: + marker = ' ' * (result.start_pos + column_offset) + '^' + error('ignoring invalid multiline annotation continuation:\n%s\n%s' % + (original_line, marker), + position) + + def _parse_annotation_options_list(self, position, column, line, options): + ''' + Parse annotation options into a list. For example:: + + ┌──────────────────────────────────────────────────────────────┐ + │ 'option1 option2 option3' │ ─▷ source + ├──────────────────────────────────────────────────────────────┤ + │ ['option1', 'option2', 'option3'] │ ◁─ parsed options + └──────────────────────────────────────────────────────────────┘ + + :param position: :class:`giscanner.message.Position` of `line` in the source file + :param column: start column of the `options` in the source file + :param line: complete source line + :param options: annotation options to parse + :returns: a list of annotation options + ''' + + parsed = [] - for tag in comment_block.tags.itervalues(): - self._clean_comment_block_part(tag) + if options: + result = options.find('=') + if result >= 0: + marker = ' ' * (column + result) + '^' + warn('invalid annotation options: expected a "list" but ' + 'received "key=value pairs":\n%s\n%s' % (line, marker), + position) + parsed = self._parse_annotation_options_unknown(position, column, line, options) + else: + parsed = options.split(' ') + + return parsed + + def _parse_annotation_options_dict(self, position, column, line, options): + ''' + Parse annotation options into a dict. For example:: + + ┌──────────────────────────────────────────────────────────────┐ + │ 'option1=value1 option2 option3=value2' │ ─▷ source + ├──────────────────────────────────────────────────────────────┤ + │ {'option1': 'value1', 'option2': None, 'option3': 'value2'} │ ◁─ parsed options + └──────────────────────────────────────────────────────────────┘ + + :param position: :class:`giscanner.message.Position` of `line` in the source file + :param column: start column of the `options` in the source file + :param line: complete source line + :param options: annotation options to parse + :returns: an ordered dictionary of annotation options + ''' + + parsed = OrderedDict() + + if options: + for p in options.split(' '): + parts = p.split('=', 1) + key = parts[0] + value = parts[1] if len(parts) == 2 else None + parsed[key] = value - for param in comment_block.params.itervalues(): - self._clean_comment_block_part(param) + return parsed - # Validate and store block. - comment_block.validate() - return comment_block + def _parse_annotation_options_unknown(self, position, column, line, options): + ''' + Parse annotation options into a list holding a single item. This is used when the + annotation options to parse in not known to be a list nor dict. For example:: + + ┌──────────────────────────────────────────────────────────────┐ + │ ' option1 option2 option3=value1 ' │ ─▷ source + ├──────────────────────────────────────────────────────────────┤ + │ ['option1 option2 option3=value1'] │ ◁─ parsed options + └──────────────────────────────────────────────────────────────┘ + + :param position: :class:`giscanner.message.Position` of `line` in the source file + :param column: start column of the `options` in the source file + :param line: complete source line + :param options: annotation options to parse + :returns: a list of annotation options + ''' + + if options: + return [options.strip()] + + def _parse_annotation(self, position, column, line, annotation): + ''' + Parse an annotation into the annotation name and a list or dict (depending on the + name of the annotation) holding the options. For example:: + + ┌──────────────────────────────────────────────────────────────┐ + │ 'name opt1=value1 opt2=value2 opt3' │ ─▷ source + ├──────────────────────────────────────────────────────────────┤ + │ 'name', {'opt1': 'value1', 'opt2':'value2', 'opt3':None} │ ◁─ parsed annotation + └──────────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────────────────────────────────┐ + │ 'name opt1 opt2' │ ─▷ source + ├──────────────────────────────────────────────────────────────┤ + │ 'name', ['opt1', 'opt2'] │ ◁─ parsed annotation + └──────────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────────────────────────────────┐ + │ 'unkownname unknown list of options' │ ─▷ source + ├──────────────────────────────────────────────────────────────┤ + │ 'unkownname', ['unknown list of options'] │ ◁─ parsed annotation + └──────────────────────────────────────────────────────────────┘ + + :param position: :class:`giscanner.message.Position` of `line` in the source file + :param column: start column of the `annotation` in the source file + :param line: complete source line + :param annotation: annotation to parse + :returns: a tuple containing the annotation name and options + ''' - def _clean_comment_block_part(self, part): - if part.comment: - part.comment = part.comment.strip() + # Transform deprecated type syntax "tokens" + annotation = annotation.replace('<', ANN_LPAR).replace('>', ANN_RPAR) + + parts = annotation.split(' ', 1) + ann_name = parts[0].lower() + ann_options = parts[1] if len(parts) == 2 else None + + if ann_name == ANN_INOUT_ALT: + marker = ' ' * (column) + '^' + warn('"%s" annotation has been deprecated, please use "%s" instead:\n%s\n%s' % + (ANN_INOUT_ALT, ANN_INOUT, line, marker), + position) + + ann_name = ANN_INOUT + elif ann_name == ANN_ATTRIBUTE: + marker = ' ' * (column) + '^' + warn('"%s" annotation has been deprecated, please use "%s" instead:\n%s\n%s' % + (ANN_ATTRIBUTE, ANN_ATTRIBUTES, line, marker), + position) + + ann_name = ANN_ATTRIBUTES + ann_options = self._parse_annotation_options_list(position, column, line, ann_options) + n_options = len(ann_options) + if n_options == 1: + ann_options = ann_options[0] + elif n_options == 2: + ann_options = '%s=%s' % (ann_options[0], ann_options[1]) + else: + marker = ' ' * (column) + '^' + error('malformed "(attribute)" annotation will be ignored:\n%s\n%s' % + (line, marker), + position) + return None, None + + column += len(ann_name) + 2 + + if ann_name in LIST_ANNOTATIONS: + ann_options = self._parse_annotation_options_list(position, column, line, ann_options) + elif ann_name in DICT_ANNOTATIONS: + ann_options = self._parse_annotation_options_dict(position, column, line, ann_options) else: - part.comment = None + ann_options = self._parse_annotation_options_unknown(position, column, line, + ann_options) + + return ann_name, ann_options - if part.value: - part.value = part.value.strip() + def _parse_annotations(self, position, column, line, fields, parse_options=True): + ''' + Parse annotations into a :class:`GtkDocAnnotations` object. + + :param position: :class:`giscanner.message.Position` of `line` in the source file + :param column: start column of the `annotations` in the source file + :param line: complete source line + :param fields: string containing the fields to parse + :param parse_options: whether options will be parsed into a :class:`GtkDocAnnotations` + object or into a :class:`list` + :returns: if `parse_options` evaluates to True a :class:`GtkDocAnnotations` object, + a :class:`list` otherwise. If `line` does not contain any annotations, + :const:`None` + ''' + + if parse_options: + parsed_annotations = GtkDocAnnotations(position) else: - part.value = '' + parsed_annotations = [] + + i = 0 + parens_level = 0 + prev_char = '' + char_buffer = [] + start_pos = 0 + end_pos = 0 + + for i, cur_char in enumerate(fields): + cur_char_is_space = cur_char.isspace() + + if cur_char == ANN_LPAR: + parens_level += 1 + + if parens_level == 1: + start_pos = i + + if prev_char == ANN_LPAR: + marker = ' ' * (column + i) + '^' + error('unexpected parentheses, annotations will be ignored:\n%s\n%s' % + (line, marker), + position) + return _ParseAnnotationsResult(False, None, None, None) + elif parens_level > 1: + char_buffer.append(cur_char) + elif cur_char == ANN_RPAR: + parens_level -= 1 + + if prev_char == ANN_LPAR: + marker = ' ' * (column + i) + '^' + error('unexpected parentheses, annotations will be ignored:\n%s\n%s' % + (line, marker), + position) + return _ParseAnnotationsResult(False, None, None, None) + elif parens_level < 0: + marker = ' ' * (column + i) + '^' + error('unbalanced parentheses, annotations will be ignored:\n%s\n%s' % + (line, marker), + position) + return _ParseAnnotationsResult(False, None, None, None) + elif parens_level == 0: + end_pos = i + 1 + + if parse_options is True: + name, options = self._parse_annotation(position, + column + start_pos, + line, + ''.join(char_buffer).strip()) + if name is not None: + if name in parsed_annotations: + marker = ' ' * (column + i) + '^' + error('multiple "%s" annotations:\n%s\n%s' % + (name, line, marker), position) + parsed_annotations[name] = options + else: + parsed_annotations.append(''.join(char_buffer).strip()) - def _validate_multiline_annotation_continuation(self, line, original_line, - column_offset, position): + char_buffer = [] + else: + char_buffer.append(cur_char) + elif cur_char_is_space: + if parens_level > 0: + char_buffer.append(cur_char) + else: + if parens_level == 0: + break + else: + char_buffer.append(cur_char) + + prev_char = cur_char + + if parens_level > 0: + marker = ' ' * (column + i) + '^' + error('unbalanced parentheses, annotations will be ignored:\n%s\n%s' % + (line, marker), + position) + return _ParseAnnotationsResult(False, None, None, None) + else: + return _ParseAnnotationsResult(True, parsed_annotations, start_pos, end_pos) + + def _parse_fields(self, position, column, line, fields, parse_options=True, + validate_description_field=True): + ''' + Parse annotations out of field data. For example:: + + ┌──────────────────────────────────────────────────────────────┐ + │ '(skip): description of some parameter │ ─▷ source + ├──────────────────────────────────────────────────────────────┤ + │ ({'skip': []}, 'description of some parameter') │ ◁─ annotations and + └──────────────────────────────────────────────────────────────┘ remaining fields + + :param position: :class:`giscanner.message.Position` of `line` in the source file + :param column: start column of `fields` in the source file + :param line: complete source line + :param fields: string containing the fields to parse + :param parse_options: whether options will be parsed into a :class:`GtkDocAnnotations` + object or into a :class:`list` + :param validate_description_field: :const:`True` to validate the description field + :returns: if `parse_options` evaluates to True a :class:`GtkDocAnnotations` object, + a :class:`list` otherwise. If `line` does not contain any annotations, + :const:`None` and a string holding the remaining fields ''' - Validate parameters and tags (except the first line) and generate - warnings about invalid annotations spanning multiple lines. + description_field = '' + result = self._parse_annotations(position, column, line, fields, parse_options) + if result.success: + description_field = fields[result.end_pos:].strip() + + if description_field and validate_description_field: + if description_field.startswith(':'): + description_field = description_field[1:] + else: + if result.end_pos > 0: + marker_position = column + result.end_pos + marker = ' ' * marker_position + '^' + warn('missing ":" at column %s:\n%s\n%s' % + (marker_position + 1, line, marker), + position) + + return _ParseFieldsResult(result.success, result.annotations, description_field) + + +class GtkDocCommentBlockWriter(object): + ''' + Serialized :class:`GtkDocCommentBlock` objects into GTK-Doc comment blocks. + ''' + + def __init__(self, indent=True): + #: :const:`True` if the original indentation preceding the "``*``" needs to be retained, + #: :const:`False` otherwise. Default value is :const:`True`. + self.indent = indent - :param line: line to validate, stripped from ' * ' at start of the line. - :param original_line: original line to validate (used in warning messages) - :param column_offset: column width of ' * ' at the time it was stripped from `line` - :param position: position of `line` in the source file + def _serialize_annotations(self, annotations): + ''' + Serialize an annotation field. For example:: + + ┌──────────────────────────────────────────────────────────────┐ + │ {'name': {'opt1': 'value1', 'opt2':'value2', 'opt3':None} │ ◁─ GtkDocAnnotations + ├──────────────────────────────────────────────────────────────┤ + │ '(name opt1=value1 opt2=value2 opt3)' │ ─▷ serialized + └──────────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────────────────────────────────┐ + │ {'name': ['opt1', 'opt2']} │ ◁─ GtkDocAnnotations + ├──────────────────────────────────────────────────────────────┤ + │ '(name opt1 opt2)' │ ─▷ serialized + └──────────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────────────────────────────────┐ + │ {'unkownname': ['unknown list of options']} │ ◁─ GtkDocAnnotations + ├──────────────────────────────────────────────────────────────┤ + │ '(unkownname unknown list of options)' │ ─▷ serialized + └──────────────────────────────────────────────────────────────┘ + + :param annotations: :class:`GtkDocAnnotations` to be serialized + :returns: a string ''' - result = MULTILINE_ANNOTATION_CONTINUATION_RE.search(line) - if result: - line = result.group('description') - column = result.start('annotations') + column_offset - marker = ' '*column + '^' - message.warn('ignoring invalid multiline annotation continuation:\n' - '%s\n%s' % (original_line, marker), - position) + serialized = [] + + for ann_name, options in annotations.items(): + if options: + if isinstance(options, list): + serialize_options = ' '.join(options) + else: + serialize_options = '' + + for key, value in options.items(): + if value: + serialize_options += '%s=%s ' % (key, value) + else: + serialize_options += '%s ' % (key, ) + + serialize_options = serialize_options.strip() + + serialized.append('(%s %s)' % (ann_name, serialize_options)) + else: + serialized.append('(%s)' % (ann_name, )) + + return ' '.join(serialized) + + def _serialize_parameter(self, parameter): + ''' + Serialize a parameter. + + :param parameter: :class:`GtkDocParameter` to be serialized + :returns: a string + ''' + + # parameter_name field + serialized = '@%s' % (parameter.name, ) + + # annotations field + if parameter.annotations: + serialized += ': ' + self._serialize_annotations(parameter.annotations) + + # description field + if parameter.description: + if parameter.description.startswith('\n'): + serialized += ':' + parameter.description + else: + serialized += ': ' + parameter.description + else: + serialized += ':' + + return serialized.split('\n') + + def _serialize_tag(self, tag): + ''' + Serialize a tag. + + :param tag: :class:`GtkDocTag` to be serialized + :returns: a string + ''' + + # tag_name field + serialized = tag.name.capitalize() + + # annotations field + if tag.annotations: + serialized += ': ' + self._serialize_annotations(tag.annotations) + + # value field + if tag.value: + serialized += ': ' + tag.value + + # description field + if tag.description: + if tag.description.startswith('\n'): + serialized += ':' + tag.description + else: + serialized += ': ' + tag.description + + if not tag.value and not tag.description: + serialized += ':' + + return serialized.split('\n') - @classmethod - def parse_options(cls, tag, value): - # (foo) - # (bar opt1 opt2 ...) - opened = -1 - options = DocOptions() - options.position = tag.position - - for i, c in enumerate(value): - if c == '(' and opened == -1: - opened = i+1 - if c == ')' and opened != -1: - segment = value[opened:i] - parts = segment.split(' ', 1) - if len(parts) == 2: - name, option = parts - elif len(parts) == 1: - name = parts[0] - option = None + def write(self, block): + ''' + Serialize a :class:`GtkDocCommentBlock` object. + + :param block: :class:`GtkDocCommentBlock` to be serialized + :returns: a string + ''' + + if block is None: + return '' + else: + lines = [] + + # Identifier part + if block.name.startswith('SECTION'): + lines.append(block.name) + else: + if block.annotations: + annotations = self._serialize_annotations(block.annotations) + lines.append('%s: %s' % (block.name, annotations)) else: - raise AssertionError - if option is not None: - option = DocOption(tag, option) - options.add(name, option) - opened = -1 + # Note: this delimiter serves no purpose other than most people being used + # to reading/writing it. It is completely legal to ommit this. + lines.append('%s:' % (block.name, )) + + # Parameter parts + for param in block.params.values(): + lines.extend(self._serialize_parameter(param)) + + # Comment block description part + if block.description: + lines.append('') + for l in block.description.split('\n'): + lines.append(l) + + # Tag parts + if block.tags: + # Note: this empty line servers no purpose other than most people being used + # to reading/writing it. It is completely legal to ommit this. + lines.append('') + for tag in block.tags.values(): + lines.extend(self._serialize_tag(tag)) + + # Restore comment block indentation and * + if self.indent: + indent = Counter(block.indentation).most_common(1)[0][0] or ' ' + if indent.endswith('\t'): + start_indent = indent + line_indent = indent + ' ' + else: + start_indent = indent[:-1] + line_indent = indent + else: + start_indent = '' + line_indent = ' ' + + i = 0 + while i < len(lines): + line = lines[i] + if line: + lines[i] = '%s* %s\n' % (line_indent, line) + else: + lines[i] = '%s*\n' % (line_indent, ) + i += 1 + + # Restore comment block start and end tokens + lines.insert(0, '%s/**\n' % (start_indent, )) + lines.append('%s*/\n' % (line_indent, )) + + # Restore code before and after comment block start and end tokens + if block.code_before: + lines.insert(0, '%s\n' % (block.code_before, )) + + if block.code_after: + lines.append('%s\n' % (block.code_after, )) - return options + return ''.join(lines) diff --git a/giscanner/annotationpatterns.py b/giscanner/annotationpatterns.py deleted file mode 100644 index a4030051..00000000 --- a/giscanner/annotationpatterns.py +++ /dev/null @@ -1,808 +0,0 @@ -# -*- Mode: Python -*- -# GObject-Introspection - a framework for introspecting GObject libraries -# Copyright (C) 2012 Dieter Verfaillie <dieterv@optionexplicit.be> -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -# 02110-1301, USA. -# - - -''' -This module provides regular expression programs that can be used to identify -and extract useful information from different parts of GTK-Doc comment blocks. -These programs are built to: - - match (or substitute) a single comment block line at a time; - - support MULTILINE mode and should support (but remains untested) - LOCALE and UNICODE modes. -''' - - -import re - - -# Program matching the start of a comment block. -# -# Results in 0 symbolic groups. -COMMENT_START_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - / # 1 forward slash character - \*{2} # exactly 2 asterisk characters - [^\S\n\r]* # 0 or more whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching the end of a comment block. -# -# Results in 0 symbolic groups. -COMMENT_END_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - \*+ # 1 or more asterisk characters - / # 1 forward slash character - $ # end - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching the "*" at the beginning of every -# line inside a comment block. -# -# Results in 0 symbolic groups. -COMMENT_ASTERISK_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - \* # 1 asterisk character - [^\S\n\r]? # 0 or 1 whitespace characters - # Carefull: removing more would - # break embedded example program - # indentation - ''', - re.VERBOSE) - -# Program matching an empty line. -# -# Results in 0 symbolic groups. -EMPTY_LINE_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or 1 whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching SECTION identifiers. -# -# Results in 2 symbolic groups: -# - group 1 = colon -# - group 2 = section_name -SECTION_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - SECTION # SECTION - [^\S\n\r]* # 0 or more whitespace characters - (?P<colon>:?) # colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<section_name>\w\S+)? # section name - [^\S\n\r]* # 0 or more whitespace characters - $ - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching symbol (function, constant, struct and enum) identifiers. -# -# Results in 3 symbolic groups: -# - group 1 = symbol_name -# - group 2 = colon -# - group 3 = annotations -SYMBOL_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - (?P<symbol_name>[\w-]*\w) # symbol name - [^\S\n\r]* # 0 or more whitespace characters - (?P<colon>:?) # colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<annotations>(?:\(.*?\)[^\S\n\r]*)*) # annotations - [^\S\n\r]* # 0 or more whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching property identifiers. -# -# Results in 4 symbolic groups: -# - group 1 = class_name -# - group 2 = property_name -# - group 3 = colon -# - group 4 = annotations -PROPERTY_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - (?P<class_name>[\w]+) # class name - [^\S\n\r]* # 0 or more whitespace characters - :{1} # required colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<property_name>[\w-]*\w) # property name - [^\S\n\r]* # 0 or more whitespace characters - (?P<colon>:?) # colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<annotations>(?:\(.*?\)[^\S\n\r]*)*) # annotations - [^\S\n\r]* # 0 or more whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching signal identifiers. -# -# Results in 4 symbolic groups: -# - group 1 = class_name -# - group 2 = signal_name -# - group 3 = colon -# - group 4 = annotations -SIGNAL_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - (?P<class_name>[\w]+) # class name - [^\S\n\r]* # 0 or more whitespace characters - :{2} # 2 required colons - [^\S\n\r]* # 0 or more whitespace characters - (?P<signal_name>[\w-]*\w) # signal name - [^\S\n\r]* # 0 or more whitespace characters - (?P<colon>:?) # colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<annotations>(?:\(.*?\)[^\S\n\r]*)*) # annotations - [^\S\n\r]* # 0 or more whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching parameters. -# -# Results in 4 symbolic groups: -# - group 1 = parameter_name -# - group 2 = annotations -# - group 3 = colon -# - group 4 = description -PARAMETER_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - @ # @ character - (?P<parameter_name>[\w-]*\w|\.\.\.) # parameter name - [^\S\n\r]* # 0 or more whitespace characters - :{1} # required colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<annotations>(?:\(.*?\)[^\S\n\r]*)*) # annotations - (?P<colon>:?) # colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<description>.*?) # description - [^\S\n\r]* # 0 or more whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching old style "Description:" tag. -# -# Results in 0 symbolic groups. -DESCRIPTION_TAG_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - Description: # 'Description:' literal - ''', - re.VERBOSE | re.MULTILINE) - -# Program matching tags. -# -# Results in 4 symbolic groups: -# - group 1 = tag_name -# - group 2 = annotations -# - group 3 = colon -# - group 4 = description -TAG_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - (?P<tag_name>virtual|since|stability| - deprecated|returns| - return\ value|attributes| - rename\ to|type| - unref\ func|ref\ func| - set\ value\ func| - get\ value\ func| - transfer|value) # tag name - [^\S\n\r]* # 0 or more whitespace characters - :{1} # required colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<annotations>(?:\(.*?\)[^\S\n\r]*)*) # annotations - (?P<colon>:?) # colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<description>.*?) # description - [^\S\n\r]* # 0 or more whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE | re.IGNORECASE) - -# Program matching multiline annotation continuations. -# This is used on multiline parameters and tags (but not on the first line) to -# generate warnings about invalid annotations spanning multiple lines. -# -# Results in 3 symbolic groups: -# - group 2 = annotations -# - group 3 = colon -# - group 4 = description -MULTILINE_ANNOTATION_CONTINUATION_RE = re.compile(r''' - ^ # start - [^\S\n\r]* # 0 or more whitespace characters - (?P<annotations>(?:\(.*?\)[^\S\n\r]*)*) # annotations - (?P<colon>:) # colon - [^\S\n\r]* # 0 or more whitespace characters - (?P<description>.*?) # description - [^\S\n\r]* # 0 or more whitespace characters - $ # end - ''', - re.VERBOSE | re.MULTILINE) - - -if __name__ == '__main__': - import unittest - - identifier_section_tests = [ - (SECTION_RE, 'TSIEOCN', - None), - (SECTION_RE, 'section', - None), - (SECTION_RE, 'section:', - None), - (SECTION_RE, 'section:test', - None), - (SECTION_RE, 'SECTION', - {'colon': '', - 'section_name': None}), - (SECTION_RE, 'SECTION \t ', - {'colon': '', - 'section_name': None}), - (SECTION_RE, ' \t SECTION \t ', - {'colon': '', - 'section_name': None}), - (SECTION_RE, 'SECTION: \t ', - {'colon': ':', - 'section_name': None}), - (SECTION_RE, 'SECTION : ', - {'colon': ':', - 'section_name': None}), - (SECTION_RE, ' SECTION : ', - {'colon': ':', - 'section_name': None}), - (SECTION_RE, 'SECTION:gtkwidget', - {'colon': ':', - 'section_name': 'gtkwidget'}), - (SECTION_RE, 'SECTION:gtkwidget ', - {'colon': ':', - 'section_name': 'gtkwidget'}), - (SECTION_RE, ' SECTION:gtkwidget', - {'colon': ':', - 'section_name': 'gtkwidget'}), - (SECTION_RE, ' SECTION:gtkwidget\t ', - {'colon': ':', - 'section_name': 'gtkwidget'}), - (SECTION_RE, 'SECTION: gtkwidget ', - {'colon': ':', - 'section_name': 'gtkwidget'}), - (SECTION_RE, 'SECTION : gtkwidget', - {'colon': ':', - 'section_name': 'gtkwidget'}), - (SECTION_RE, 'SECTION gtkwidget \f ', - {'colon': '', - 'section_name': 'gtkwidget'})] - - identifier_symbol_tests = [ - (SYMBOL_RE, 'GBaseFinalizeFunc:', - {'colon': ':', - 'symbol_name': 'GBaseFinalizeFunc', - 'annotations': ''}), - (SYMBOL_RE, 'gtk_widget_show ', - {'colon': '', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, ' gtk_widget_show', - {'colon': '', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, ' gtk_widget_show ', - {'colon': '', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, 'gtk_widget_show:', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, 'gtk_widget_show :', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, 'gtk_widget_show: ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, 'gtk_widget_show : ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, ' gtk_widget_show:', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, ' gtk_widget_show :', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, ' gtk_widget_show: ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, ' gtk_widget_show : ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': ''}), - (SYMBOL_RE, 'gtk_widget_show (skip)', - {'colon': '', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, 'gtk_widget_show: (skip)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, 'gtk_widget_show : (skip)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, 'gtk_widget_show: (skip)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, 'gtk_widget_show : (skip)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, ' gtk_widget_show:(skip)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, ' gtk_widget_show :(skip)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, ' gtk_widget_show: (skip)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)'}), - (SYMBOL_RE, ' gtk_widget_show : (skip) \t ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) \t '}), - (SYMBOL_RE, ' gtk_widget_show : (skip) \t ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) \t '}), - (SYMBOL_RE, 'gtk_widget_show:(skip)(test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)(test1)'}), - (SYMBOL_RE, 'gtk_widget_show (skip)(test1)', - {'colon': '', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip)(test1)'}), - (SYMBOL_RE, 'gtk_widget_show: (skip) (test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1)'}), - (SYMBOL_RE, 'gtk_widget_show : (skip) (test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1)'}), - (SYMBOL_RE, 'gtk_widget_show: (skip) (test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1)'}), - (SYMBOL_RE, 'gtk_widget_show : (skip) (test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1)'}), - (SYMBOL_RE, ' gtk_widget_show:(skip) (test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1)'}), - (SYMBOL_RE, ' gtk_widget_show :(skip) (test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1)'}), - (SYMBOL_RE, ' gtk_widget_show: (skip) (test1)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1)'}), - (SYMBOL_RE, ' gtk_widget_show : (skip) (test1) ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) '}), - (SYMBOL_RE, 'gtk_widget_show: (skip) (test1) (test-2)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2)'}), - (SYMBOL_RE, 'gtk_widget_show : (skip) (test1) (test-2)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2)'}), - (SYMBOL_RE, 'gtk_widget_show: (skip) (test1) (test-2)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2)'}), - (SYMBOL_RE, 'gtk_widget_show : (skip) (test1) (test-2)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2)'}), - (SYMBOL_RE, ' gtk_widget_show:(skip) (test1) (test-2)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2)'}), - (SYMBOL_RE, ' gtk_widget_show :(skip) (test1) (test-2)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2)'}), - (SYMBOL_RE, ' gtk_widget_show: (skip) (test1) (test-2)', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2)'}), - (SYMBOL_RE, ' gtk_widget_show : (skip) (test1) (test-2) ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2) '}), - (SYMBOL_RE, ' gtk_widget_show : (skip) (test1) (test-2) ', - {'colon': ':', - 'symbol_name': 'gtk_widget_show', - 'annotations': '(skip) (test1) (test-2) '}), - # constants - (SYMBOL_RE, 'MY_CONSTANT:', - {'colon': ':', - 'symbol_name': 'MY_CONSTANT', - 'annotations': ''}), - # structs - (SYMBOL_RE, 'FooWidget:', - {'colon': ':', - 'symbol_name': 'FooWidget', - 'annotations': ''}), - # enums - (SYMBOL_RE, 'Something:', - {'colon': ':', - 'symbol_name': 'Something', - 'annotations': ''})] - - identifier_property_tests = [ - # simple property name - (PROPERTY_RE, 'GtkWidget:name (skip)', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': '', - 'annotations': '(skip)'}), - (PROPERTY_RE, 'GtkWidget:name', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, ' GtkWidget :name', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, 'GtkWidget: name ', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, ' GtkWidget : name ', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, 'GtkWidget:name:', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'GtkWidget:name: ', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, ' GtkWidget:name:', - {'class_name': 'GtkWidget', - 'property_name': 'name', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'Something:name:', - {'class_name': 'Something', - 'property_name': 'name', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'Something:name: ', - {'class_name': 'Something', - 'property_name': 'name', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, ' Something:name:', - {'class_name': 'Something', - 'property_name': 'name', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'Weird-thing:name:', - None), - (PROPERTY_RE, 'really-weird_thing:name:', - None), - (PROPERTY_RE, 'GWin32InputStream:handle:', - {'class_name': 'GWin32InputStream', - 'property_name': 'handle', - 'colon': ':', - 'annotations': ''}), - # property name that contains a dash - (PROPERTY_RE, 'GtkWidget:double-buffered (skip)', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': '', - 'annotations': '(skip)'}), - (PROPERTY_RE, 'GtkWidget:double-buffered', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, ' GtkWidget :double-buffered', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, 'GtkWidget: double-buffered ', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, ' GtkWidget : double-buffered ', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': '', - 'annotations': ''}), - (PROPERTY_RE, 'GtkWidget:double-buffered:', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'GtkWidget:double-buffered: ', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, ' GtkWidget:double-buffered:', - {'class_name': 'GtkWidget', - 'property_name': 'double-buffered', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'Something:double-buffered:', - {'class_name': 'Something', - 'property_name': 'double-buffered', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'Something:double-buffered: ', - {'class_name': 'Something', - 'property_name': 'double-buffered', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, ' Something:double-buffered:', - {'class_name': 'Something', - 'property_name': 'double-buffered', - 'colon': ':', - 'annotations': ''}), - (PROPERTY_RE, 'Weird-thing:double-buffered:', - None), - (PROPERTY_RE, 'really-weird_thing:double-buffered:', - None), - (PROPERTY_RE, ' GMemoryOutputStream:realloc-function: (skip)', - {'class_name': 'GMemoryOutputStream', - 'property_name': 'realloc-function', - 'colon': ':', - 'annotations': '(skip)'})] - - identifier_signal_tests = [ - # simple property name - (SIGNAL_RE, 'GtkWidget::changed: (skip)', - {'class_name': 'GtkWidget', - 'signal_name': 'changed', - 'colon': ':', - 'annotations': '(skip)'}), - (SIGNAL_RE, 'GtkWidget::changed:', - {'class_name': 'GtkWidget', - 'signal_name': 'changed', - 'colon': ':', - 'annotations': ''}), - (SIGNAL_RE, 'Something::changed:', - {'class_name': 'Something', - 'signal_name': 'changed', - 'colon': ':', - 'annotations': ''}), - (SIGNAL_RE, 'Weird-thing::changed:', - None), - (SIGNAL_RE, 'really-weird_thing::changed:', - None), - # property name that contains a dash - (SIGNAL_RE, 'GtkWidget::hierarchy-changed: (skip)', - {'class_name': 'GtkWidget', - 'signal_name': 'hierarchy-changed', - 'colon': ':', - 'annotations': '(skip)'}), - (SIGNAL_RE, 'GtkWidget::hierarchy-changed:', - {'class_name': 'GtkWidget', - 'signal_name': 'hierarchy-changed', - 'colon': ':', - 'annotations': ''}), - (SIGNAL_RE, 'Something::hierarchy-changed:', - {'class_name': 'Something', - 'signal_name': 'hierarchy-changed', - 'colon': ':', - 'annotations': ''}), - (SIGNAL_RE, 'Weird-thing::hierarchy-changed:', - None), - (SIGNAL_RE, 'really-weird_thing::hierarchy-changed:', - None)] - - parameter_tests = [ - (PARAMETER_RE, '@Short_description: Base class for all widgets ', - {'parameter_name': 'Short_description', - 'annotations': '', - 'colon': '', - 'description': 'Base class for all widgets'}), - (PARAMETER_RE, '@...: the value of the first property, followed optionally by more', - {'parameter_name': '...', - 'annotations': '', - 'colon': '', - 'description': 'the value of the first property, followed optionally by more'}), - (PARAMETER_RE, '@widget: a #GtkWidget', - {'parameter_name': 'widget', - 'annotations': '', - 'colon': '', - 'description': 'a #GtkWidget'}), - (PARAMETER_RE, '@widget_pointer: (inout) (transfer none): ' - 'address of a variable that contains @widget', - {'parameter_name': 'widget_pointer', - 'annotations': '(inout) (transfer none)', - 'colon': ':', - 'description': 'address of a variable that contains @widget'}), - (PARAMETER_RE, '@weird_thing: (inout) (transfer none) (allow-none) (attribute) (destroy) ' - '(foreign) (inout) (out) (transfer) (skip) (method): some weird @thing', - {'parameter_name': 'weird_thing', - 'annotations': '(inout) (transfer none) (allow-none) (attribute) (destroy) ' - '(foreign) (inout) (out) (transfer) (skip) (method)', - 'colon': ':', - 'description': 'some weird @thing'}), - (PARAMETER_RE, '@data: a pointer to the element data. The data may be moved as elements ' - 'are added to the #GByteArray.', - {'parameter_name': 'data', - 'annotations': '', - 'colon': '', - 'description': 'a pointer to the element data. The data may be moved as elements ' - 'are added to the #GByteArray.'}), - (PARAMETER_RE, '@a: a #GSequenceIter', - {'parameter_name': 'a', - 'annotations': '', - 'colon': '', - 'description': 'a #GSequenceIter'}), - (PARAMETER_RE, '@keys: (array length=n_keys) (element-type GQuark) (allow-none):', - {'parameter_name': 'keys', - 'annotations': '(array length=n_keys) (element-type GQuark) (allow-none)', - 'colon': ':', - 'description': ''})] - - tag_tests = [ - (TAG_RE, 'Since 3.0', - None), - (TAG_RE, 'Since: 3.0', - {'tag_name': 'Since', - 'annotations': '', - 'colon': '', - 'description': '3.0'}), - (TAG_RE, 'Attributes: (inout) (transfer none): some note about attributes', - {'tag_name': 'Attributes', - 'annotations': '(inout) (transfer none)', - 'colon': ':', - 'description': 'some note about attributes'}), - (TAG_RE, 'Rename to: something_else', - {'tag_name': 'Rename to', - 'annotations': '', - 'colon': '', - 'description': 'something_else'}), - (TAG_RE, '@Deprecated: Since 2.8, reference counting is done atomically', - None), - (TAG_RE, 'Returns %TRUE and does weird things', - None), - (TAG_RE, 'Returns: a #GtkWidget', - {'tag_name': 'Returns', - 'annotations': '', - 'colon': '', - 'description': 'a #GtkWidget'}), - (TAG_RE, 'Return value: (transfer none): The binary data that @text responds. ' - 'This pointer', - {'tag_name': 'Return value', - 'annotations': '(transfer none)', - 'colon': ':', - 'description': 'The binary data that @text responds. This pointer'}), - (TAG_RE, 'Return value: (transfer full) (array length=out_len) (element-type guint8):', - {'tag_name': 'Return value', - 'annotations': '(transfer full) (array length=out_len) (element-type guint8)', - 'colon': ':', - 'description': ''}), - (TAG_RE, 'Returns: A boolean value, but let me tell you a bit about this boolean. It', - {'tag_name': 'Returns', - 'annotations': '', - 'colon': '', - 'description': 'A boolean value, but let me tell you a bit about this boolean. ' - 'It'}), - (TAG_RE, 'Returns: (transfer container) (element-type GObject.ParamSpec): a', - {'tag_name': 'Returns', - 'annotations': '(transfer container) (element-type GObject.ParamSpec)', - 'colon': ':', - 'description': 'a'}), - (TAG_RE, 'Return value: (type GLib.HashTable<utf8,GLib.HashTable<utf8,utf8>>) ' - '(transfer full):', - {'tag_name': 'Return value', - 'annotations': '(type GLib.HashTable<utf8,GLib.HashTable<utf8,utf8>>) ' - '(transfer full)', - 'colon': ':', - 'description': ''})] - - - def create_tests(cls, test_name, testcases): - for (index, testcase) in enumerate(testcases): - real_test_name = '%s_%03d' % (test_name, index) - - test_method = cls.__create_test__(testcase) - test_method.__name__ = real_test_name - setattr(cls, real_test_name, test_method) - - - class TestProgram(unittest.TestCase): - @classmethod - def __create_test__(cls, testcase): - def do_test(self): - (program, text, expected) = testcase - - match = program.search(text) - - if expected is None: - msg = 'Program matched text but shouldn\'t:\n"%s"' - self.assertTrue(match is None, msg % (text)) - else: - msg = 'Program should match text but didn\'t:\n"%s"' - self.assertTrue(match is not None, msg % (text)) - - for key, value in expected.items(): - msg = 'expected "%s" for "%s" but match returned "%s"' - msg = msg % (value, key, match.group(key)) - self.assertEqual(match.group(key), value, msg) - - return do_test - - - # Create tests from data - create_tests(TestProgram, 'test_identifier_section', identifier_section_tests) - create_tests(TestProgram, 'test_identifier_symbol', identifier_symbol_tests) - create_tests(TestProgram, 'test_identifier_property', identifier_property_tests) - create_tests(TestProgram, 'test_identifier_signal', identifier_signal_tests) - create_tests(TestProgram, 'test_parameter', parameter_tests) - create_tests(TestProgram, 'test_tag', tag_tests) - - # Run test suite - unittest.main() diff --git a/giscanner/ast.py b/giscanner/ast.py index 654c68e2..b5b2ad71 100644 --- a/giscanner/ast.py +++ b/giscanner/ast.py @@ -20,22 +20,25 @@ # import copy +from itertools import chain from . import message +from .collections import OrderedDict from .message import Position -from .odict import odict from .utils import to_underscores + class Type(object): - """A Type can be either: -* A reference to a node (target_giname) -* A reference to a "fundamental" type like 'utf8' -* A "foreign" type - this can be any string." -If none are specified, then it's in an "unresolved" state. An -unresolved type can have two data sources; a "ctype" which comes -from a C type string, or a gtype_name (from g_type_name()). -""" # ''' + """ + A Type can be either: + * A reference to a node (target_giname) + * A reference to a "fundamental" type like 'utf8' + * A "foreign" type - this can be any string." + If none are specified, then it's in an "unresolved" state. An + unresolved type can have two data sources; a "ctype" which comes + from a C type string, or a gtype_name (from g_type_name()). + """ def __init__(self, ctype=None, @@ -83,6 +86,8 @@ from a C type string, or a gtype_name (from g_type_name()). return self.ctype elif self.gtype_name: return self.gtype_name + elif self.target_giname: + return self.target_giname else: assert False @@ -94,7 +99,8 @@ in contrast to the other create_type() functions.""" # First, is it a fundamental? fundamental = type_names.get(gtype_name) if fundamental is not None: - return cls(target_fundamental=fundamental.target_fundamental) + return cls(target_fundamental=fundamental.target_fundamental, + ctype=fundamental.ctype) if gtype_name == 'GHashTable': return Map(TYPE_ANY, TYPE_ANY, gtype_name=gtype_name) elif gtype_name in ('GArray', 'GPtrArray', 'GByteArray'): @@ -121,11 +127,12 @@ in contrast to the other create_type() functions.""" def __cmp__(self, other): if self.target_fundamental: return cmp(self.target_fundamental, other.target_fundamental) - if self.target_giname: + elif self.target_giname: return cmp(self.target_giname, other.target_giname) - if self.target_foreign: + elif self.target_foreign: return cmp(self.target_foreign, other.target_foreign) - return cmp(self.ctype, other.ctype) + else: + return cmp(self.ctype, other.ctype) def is_equiv(self, typeval): """Return True if the specified types are compatible at @@ -166,6 +173,7 @@ in contrast to the other create_type() functions.""" data = '' return '%s(%sctype=%s)' % (self.__class__.__name__, data, self.ctype) + class TypeUnknown(Type): def __init__(self): Type.__init__(self, _target_unknown=True) @@ -347,9 +355,7 @@ SIGNAL_MUST_COLLECT = 'must-collect' class Namespace(object): - def __init__(self, name, version, - identifier_prefixes=None, - symbol_prefixes=None): + def __init__(self, name, version, identifier_prefixes=None, symbol_prefixes=None): self.name = name self.version = version if identifier_prefixes is not None: @@ -363,11 +369,15 @@ class Namespace(object): self.symbol_prefixes = [to_underscores(p).lower() for p in ps] # cache upper-cased versions self._ucase_symbol_prefixes = [p.upper() for p in self.symbol_prefixes] - self.names = odict() # Maps from GIName -> node - self.aliases = {} # Maps from GIName -> GIName - self.type_names = {} # Maps from GTName -> node - self.ctypes = {} # Maps from CType -> node - self.symbols = {} # Maps from function symbols -> Function + self.names = OrderedDict() # Maps from GIName -> node + self.aliases = {} # Maps from GIName -> GIName + self.type_names = {} # Maps from GTName -> node + self.ctypes = {} # Maps from CType -> node + self.symbols = {} # Maps from function symbols -> Function + self.includes = set() # Include + self.shared_libraries = [] # str + self.c_includes = [] # str + self.exported_packages = [] # str def type_from_name(self, name, ctype=None): """Backwards compatibility method for older .gir files, which @@ -399,6 +409,24 @@ but adds it to things like ctypes, symbols, and type_names. self.type_names[node.gtype_name] = node elif isinstance(node, Function): self.symbols[node.symbol] = node + if isinstance(node, (Compound, Class, Interface)): + for fn in chain(node.methods, node.static_methods, node.constructors): + if not isinstance(fn, Function): + continue + fn.namespace = self + self.symbols[fn.symbol] = fn + if isinstance(node, (Class, Interface)): + for m in chain(node.signals, node.properties): + m.namespace = self + if isinstance(node, (Enum, Bitfield)): + for fn in node.static_methods: + if not isinstance(fn, Function): + continue + fn.namespace = self + self.symbols[fn.symbol] = fn + for member in node.members: + member.namespace = self + self.symbols[member.symbol] = member if hasattr(node, 'ctype'): self.ctypes[node.ctype] = node @@ -456,6 +484,7 @@ functions via get_by_symbol().""" for node in self.itervalues(): node.walk(callback, []) + class Include(object): def __init__(self, name, version): @@ -478,19 +507,23 @@ class Include(object): def __str__(self): return '%s-%s' % (self.name, self.version) + class Annotated(object): """An object which has a few generic metadata properties.""" def __init__(self): self.version = None + self.version_doc = None self.skip = False self.introspectable = True - self.attributes = [] # (key, value)* + self.attributes = OrderedDict() self.stability = None + self.stability_doc = None self.deprecated = None - self.deprecated_version = None + self.deprecated_doc = None self.doc = None + class Node(Annotated): """A node is a type of object which is uniquely identified by its (namespace, name) pair. When combined with a ., this is called a @@ -501,10 +534,21 @@ GIName. It's possible for nodes to contain or point to other nodes.""" def __init__(self, name=None): Annotated.__init__(self) - self.namespace = None # Should be set later by Namespace.append() + self.namespace = None # Should be set later by Namespace.append() self.name = name self.foreign = False self.file_positions = set() + self._parent = None + + def _get_parent(self): + if self._parent is not None: + return self._parent + else: + return self.namespace + + def _set_parent(self, value): + self._parent = value + parent = property(_get_parent, _set_parent) def create_type(self): """Create a Type object referencing this node.""" @@ -559,8 +603,16 @@ class Callable(Node): self.retval = retval self.parameters = parameters self.throws = not not throws - self.instance_parameter = None # Parameter - self.parent = None # A Class or Interface + self.instance_parameter = None # Parameter + self.parent = None # A Class or Interface + + # Returns all parameters, including the instance parameter + @property + def all_parameters(self): + if self.instance_parameter is not None: + return [self.instance_parameter] + self.parameters + else: + return self.parameters def get_parameter_index(self, name): for i, parameter in enumerate(self.parameters): @@ -569,7 +621,7 @@ class Callable(Node): raise ValueError("Unknown argument %s" % (name, )) def get_parameter(self, name): - for parameter in self.parameters: + for parameter in self.all_parameters: if parameter.argname == name: return parameter raise ValueError("Unknown argument %s" % (name, )) @@ -582,9 +634,10 @@ class Function(Callable): self.symbol = symbol self.is_method = False self.is_constructor = False - self.shadowed_by = None # C symbol string - self.shadows = None # C symbol string - self.moved_to = None # namespaced function name string + self.shadowed_by = None # C symbol string + self.shadows = None # C symbol string + self.moved_to = None # namespaced function name string + self.internal_skipped = False # if True, this func will not be written to GIR def clone(self): clone = copy.copy(self) @@ -595,9 +648,8 @@ class Function(Callable): def is_type_meta_function(self): # Named correctly - if not (self.name.endswith('_get_type') or - self.name.endswith('_get_gtype')): - return False + if not (self.name.endswith('_get_type') or self.name.endswith('_get_gtype')): + return False # Doesn't have any parameters if self.parameters: @@ -605,14 +657,13 @@ class Function(Callable): # Returns GType rettype = self.retval.type - if (not rettype.is_equiv(TYPE_GTYPE) and - rettype.target_giname != 'Gtk.Type'): - message.warn("function '%s' returns '%r', not a GType" % - (self.name, rettype)) + if (not rettype.is_equiv(TYPE_GTYPE) and rettype.target_giname != 'Gtk.Type'): + message.warn("function '%s' returns '%r', not a GType" % (self.name, rettype)) return False return True + class ErrorQuarkFunction(Function): def __init__(self, name, retval, parameters, throws, symbol, error_domain): @@ -633,7 +684,6 @@ class VFunction(Callable): return obj - class Varargs(Type): def __init__(self): @@ -669,6 +719,7 @@ class Array(Type): arr.size = self.size return arr + class List(Type): def __init__(self, name, element_type, **kwargs): @@ -681,6 +732,7 @@ class List(Type): def clone(self): return List(self.name, self.element_type) + class Map(Type): def __init__(self, key_type, value_type, **kwargs): @@ -693,6 +745,7 @@ class Map(Type): def clone(self): return Map(self.key_type, self.value_type) + class Alias(Node): def __init__(self, name, target, ctype=None): @@ -751,6 +804,8 @@ class Enum(Node, Registered): self.c_symbol_prefix = c_symbol_prefix self.ctype = ctype self.members = members + for member in members: + member.parent = self # Associated error domain name self.error_domain = None self.static_methods = [] @@ -772,6 +827,8 @@ class Bitfield(Node, Registered): self.ctype = ctype self.c_symbol_prefix = c_symbol_prefix self.members = members + for member in members: + member.parent = self self.static_methods = [] def _walk(self, callback, chain): @@ -787,10 +844,13 @@ class Member(Annotated): self.value = value self.symbol = symbol self.nick = nick + self.parent = None def __cmp__(self, other): return cmp(self.name, other.name) + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.name) class Compound(Node, Registered): @@ -799,7 +859,8 @@ class Compound(Node, Registered): gtype_name=None, get_type=None, c_symbol_prefix=None, - disguised=False): + disguised=False, + tag_name=None): Node.__init__(self, name) Registered.__init__(self, gtype_name, get_type) self.ctype = ctype @@ -811,6 +872,7 @@ class Compound(Node, Registered): self.gtype_name = gtype_name self.get_type = get_type self.c_symbol_prefix = c_symbol_prefix + self.tag_name = tag_name def add_gtype(self, gtype_name, get_type): self.gtype_name = gtype_name @@ -828,6 +890,19 @@ class Compound(Node, Registered): if field.anonymous_node is not None: field.anonymous_node.walk(callback, chain) + def get_field(self, name): + for field in self.fields: + if field.name == name: + return field + raise ValueError("Unknown field %s" % (name, )) + + def get_field_index(self, name): + for i, field in enumerate(self.fields): + if field.name == name: + return i + raise ValueError("Unknown field %s" % (name, )) + + class Field(Annotated): def __init__(self, name, typenode, readable, writable, bits=None, @@ -841,10 +916,14 @@ class Field(Annotated): self.bits = bits self.anonymous_node = anonymous_node self.private = False + self.parent = None # a compound def __cmp__(self, other): return cmp(self.name, other.name) + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.name) + class Record(Compound): @@ -853,13 +932,15 @@ class Record(Compound): gtype_name=None, get_type=None, c_symbol_prefix=None, - disguised=False): + disguised=False, + tag_name=None): Compound.__init__(self, name, ctype=ctype, gtype_name=gtype_name, get_type=get_type, c_symbol_prefix=c_symbol_prefix, - disguised=disguised) + disguised=disguised, + tag_name=tag_name) # If non-None, this record defines the FooClass C structure # for some Foo GObject (or similar for GInterface) self.is_gtype_struct_for = None @@ -872,13 +953,15 @@ class Union(Compound): gtype_name=None, get_type=None, c_symbol_prefix=None, - disguised=False): + disguised=False, + tag_name=None): Compound.__init__(self, name, ctype=ctype, gtype_name=gtype_name, get_type=get_type, c_symbol_prefix=c_symbol_prefix, - disguised=disguised) + disguised=disguised, + tag_name=tag_name) class Boxed(Node, Registered): @@ -922,7 +1005,7 @@ class Signal(Callable): class Class(Node, Registered): - def __init__(self, name, parent, + def __init__(self, name, parent_type, ctype=None, gtype_name=None, get_type=None, @@ -932,7 +1015,7 @@ class Class(Node, Registered): Registered.__init__(self, gtype_name, get_type) self.ctype = ctype self.c_symbol_prefix = c_symbol_prefix - self.parent = parent + self.parent_type = parent_type self.fundamental = False self.unref_func = None self.ref_func = None @@ -967,11 +1050,13 @@ class Class(Node, Registered): field.anonymous_node.walk(callback, chain) for sig in self.signals: sig.walk(callback, chain) + for prop in self.properties: + prop.walk(callback, chain) class Interface(Node, Registered): - def __init__(self, name, parent, + def __init__(self, name, parent_type, ctype=None, gtype_name=None, get_type=None, @@ -980,7 +1065,7 @@ class Interface(Node, Registered): Registered.__init__(self, gtype_name, get_type) self.ctype = ctype self.c_symbol_prefix = c_symbol_prefix - self.parent = parent + self.parent_type = parent_type self.parent_chain = [] self.methods = [] self.signals = [] @@ -990,6 +1075,9 @@ class Interface(Node, Registered): self.properties = [] self.fields = [] self.prerequisites = [] + # Not used yet, exists just to avoid an exception in + # Namespace.append() + self.constructors = [] def _walk(self, callback, chain): for meth in self.methods: @@ -1028,7 +1116,7 @@ class Property(Node): self.transfer = PARAM_TRANSFER_NONE else: self.transfer = transfer - self.parent = None # A Class or Interface + self.parent = None # A Class or Interface class Callback(Callable): diff --git a/giscanner/cachestore.py b/giscanner/cachestore.py index 44e3b04c..ad4c7a36 100644 --- a/giscanner/cachestore.py +++ b/giscanner/cachestore.py @@ -31,6 +31,7 @@ import giscanner _CACHE_VERSION_FILENAME = '.cache-version' + def _get_versionhash(): toplevel = os.path.dirname(giscanner.__file__) # Use pyc instead of py to avoid extra IO @@ -40,6 +41,7 @@ def _get_versionhash(): mtimes = (str(os.stat(source).st_mtime) for source in sources) return hashlib.sha1(''.join(mtimes)).hexdigest() + def _get_cachedir(): if 'GI_SCANNER_DISABLE_CACHE' in os.environ: return None @@ -73,7 +75,7 @@ class CacheStore(object): def __init__(self): try: self._directory = _get_cachedir() - except OSError, e: + except OSError as e: if e.errno != errno.EPERM: raise self._directory = None @@ -88,7 +90,7 @@ class CacheStore(object): version = os.path.join(self._directory, _CACHE_VERSION_FILENAME) try: cache_hash = open(version).read() - except IOError, e: + except IOError as e: # File does not exist if e.errno == errno.ENOENT: cache_hash = 0 @@ -101,7 +103,7 @@ class CacheStore(object): self._clean() try: fp = open(version, 'w') - except IOError, e: + except IOError as e: # Permission denied if e.errno == errno.EACCES: return @@ -126,13 +128,13 @@ class CacheStore(object): def _remove_filename(self, filename): try: os.unlink(filename) - except IOError, e: + except IOError as e: # Permission denied if e.errno == errno.EACCES: return else: raise - except OSError, e: + except OSError as e: # File does not exist if e.errno == errno.ENOENT: return @@ -150,14 +152,13 @@ class CacheStore(object): if store_filename is None: return - if (os.path.exists(store_filename) and - self._cache_is_valid(store_filename, filename)): + if (os.path.exists(store_filename) and self._cache_is_valid(store_filename, filename)): return None tmp_fd, tmp_filename = tempfile.mkstemp(prefix='g-ir-scanner-cache-') try: cPickle.dump(data, os.fdopen(tmp_fd, 'w')) - except IOError, e: + except IOError as e: # No space left on device if e.errno == errno.ENOSPC: self._remove_filename(tmp_filename) @@ -167,7 +168,7 @@ class CacheStore(object): try: shutil.move(tmp_filename, store_filename) - except IOError, e: + except IOError as e: # Permission denied if e.errno == errno.EACCES: self._remove_filename(tmp_filename) @@ -180,7 +181,7 @@ class CacheStore(object): return try: fd = open(store_filename) - except IOError, e: + except IOError as e: if e.errno == errno.ENOENT: return None else: diff --git a/giscanner/codegen.py b/giscanner/codegen.py index b73a7da3..e9ed9415 100644 --- a/giscanner/codegen.py +++ b/giscanner/codegen.py @@ -24,6 +24,7 @@ from contextlib import contextmanager from . import ast + class CCodeGenerator(object): def __init__(self, namespace, out_h_filename, out_c_filename): self.out_h_filename = out_h_filename @@ -36,15 +37,16 @@ class CCodeGenerator(object): return '%s_%s' % (self.namespace.symbol_prefixes[0], name) def _typecontainer_to_ctype(self, param): - if (isinstance(param, ast.Parameter) and - param.direction in (ast.PARAM_DIRECTION_OUT, - ast.PARAM_DIRECTION_INOUT)): + if (isinstance(param, ast.Parameter) + and param.direction in (ast.PARAM_DIRECTION_OUT, ast.PARAM_DIRECTION_INOUT)): suffix = '*' else: suffix = '' - if (param.type.is_equiv((ast.TYPE_STRING, ast.TYPE_FILENAME)) and - param.transfer == ast.PARAM_TRANSFER_NONE): + + if (param.type.is_equiv((ast.TYPE_STRING, ast.TYPE_FILENAME)) + and param.transfer == ast.PARAM_TRANSFER_NONE): return "const gchar*" + suffix + return param.type.ctype + suffix def _write_prelude(self, out, func): diff --git a/giscanner/collections/__init__.py b/giscanner/collections/__init__.py new file mode 100644 index 00000000..aa3814a7 --- /dev/null +++ b/giscanner/collections/__init__.py @@ -0,0 +1,23 @@ +# -*- Mode: Python -*- +# GObject-Introspection - a framework for introspecting GObject libraries +# Copyright (C) 2013 Dieter Verfaillie <dieterv@optionexplicit.be> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + + +from .counter import Counter +from .ordereddict import OrderedDict diff --git a/giscanner/collections/counter.py b/giscanner/collections/counter.py new file mode 100644 index 00000000..b337ab3f --- /dev/null +++ b/giscanner/collections/counter.py @@ -0,0 +1,305 @@ +# -*- Mode: Python -*- +# GObject-Introspection - a framework for introspecting GObject libraries +# Copyright (C) 2013 Dieter Verfaillie <dieterv@optionexplicit.be> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + + +from __future__ import absolute_import + + +try: + from collections import Counter +except ImportError: + # collections.Counter for Python 2.6, backported from + # http://hg.python.org/cpython/file/d047928ae3f6/Lib/collections/__init__.py#l402 + + from operator import itemgetter + from heapq import nlargest + from itertools import repeat, ifilter + + class Counter(dict): + '''Dict subclass for counting hashable items. Sometimes called a bag + or multiset. Elements are stored as dictionary keys and their counts + are stored as dictionary values. + + >>> c = Counter('abcdeabcdabcaba') # count elements from a string + + >>> c.most_common(3) # three most common elements + [('a', 5), ('b', 4), ('c', 3)] + >>> sorted(c) # list all unique elements + ['a', 'b', 'c', 'd', 'e'] + >>> ''.join(sorted(c.elements())) # list elements with repetitions + 'aaaaabbbbcccdde' + >>> sum(c.values()) # total of all counts + 15 + + >>> c['a'] # count of letter 'a' + 5 + >>> for elem in 'shazam': # update counts from an iterable + ... c[elem] += 1 # by adding 1 to each element's count + >>> c['a'] # now there are seven 'a' + 7 + >>> del c['b'] # remove all 'b' + >>> c['b'] # now there are zero 'b' + 0 + + >>> d = Counter('simsalabim') # make another counter + >>> c.update(d) # add in the second counter + >>> c['a'] # now there are nine 'a' + 9 + + >>> c.clear() # empty the counter + >>> c + Counter() + + Note: If a count is set to zero or reduced to zero, it will remain + in the counter until the entry is deleted or the counter is cleared: + + >>> c = Counter('aaabbc') + >>> c['b'] -= 2 # reduce the count of 'b' by two + >>> c.most_common() # 'b' is still in, but its count is zero + [('a', 3), ('c', 1), ('b', 0)] + + ''' + # References: + # http://en.wikipedia.org/wiki/Multiset + # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html + # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm + # http://code.activestate.com/recipes/259174/ + # Knuth, TAOCP Vol. II section 4.6.3 + + def __init__(self, iterable=None, **kwds): + '''Create a new, empty Counter object. And if given, count elements + from an input iterable. Or, initialize the count from another mapping + of elements to their counts. + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping + >>> c = Counter(a=4, b=2) # a new counter from keyword args + + ''' + self.update(iterable, **kwds) + + def __missing__(self, key): + 'The count of elements not in the Counter is zero.' + # Needed so that self[missing_item] does not raise KeyError + return 0 + + def most_common(self, n=None): + '''List the n most common elements and their counts from the most + common to the least. If n is None, then list all element counts. + + >>> Counter('abcdeabcdabcaba').most_common(3) + [('a', 5), ('b', 4), ('c', 3)] + + ''' + # Emulate Bag.sortedByCount from Smalltalk + if n is None: + return sorted(self.iteritems(), key=itemgetter(1), reverse=True) + return nlargest(n, self.iteritems(), key=itemgetter(1)) + + def elements(self): + '''Iterator over elements repeating each as many times as its count. + + >>> c = Counter('ABCABC') + >>> sorted(c.elements()) + ['A', 'A', 'B', 'B', 'C', 'C'] + + # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) + >>> product = 1 + >>> for factor in prime_factors.elements(): # loop over factors + ... product *= factor # and multiply them + >>> product + 1836 + + Note, if an element's count has been set to zero or is a negative + number, elements() will ignore it. + + ''' + # Emulate Bag.do from Smalltalk and Multiset.begin from C++. + for elem, count in self.iteritems(): + for _ in repeat(None, count): + yield elem + + # Override dict methods where necessary + + @classmethod + def fromkeys(cls, iterable, v=None): + # There is no equivalent method for counters because setting v=1 + # means that no element can have a count greater than one. + raise NotImplementedError( + 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') + + def update(self, iterable=None, **kwds): + '''Like dict.update() but add counts instead of replacing them. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.update('witch') # add elements from another iterable + >>> d = Counter('watch') + >>> c.update(d) # add elements from another counter + >>> c['h'] # four 'h' in which, witch, and watch + 4 + + ''' + # The regular dict.update() operation makes no sense here because the + # replace behavior results in the some of original untouched counts + # being mixed-in with all of the other counts for a mismash that + # doesn't have a straight-forward interpretation in most counting + # contexts. Instead, we implement straight-addition. Both the inputs + # and outputs are allowed to contain zero and negative counts. + + if iterable is not None: + if hasattr(iterable, 'iteritems'): + if self: + self_get = self.get + for elem, count in iterable.iteritems(): + self[elem] = self_get(elem, 0) + count + else: + dict.update(self, iterable) # fast path when counter is empty + else: + self_get = self.get + for elem in iterable: + self[elem] = self_get(elem, 0) + 1 + if kwds: + self.update(kwds) + + def subtract(self, iterable=None, **kwds): + '''Like dict.update() but subtracts counts instead of replacing them. + Counts can be reduced below zero. Both the inputs and outputs are + allowed to contain zero and negative counts. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.subtract('witch') # subtract elements from another iterable + >>> c.subtract(Counter('watch')) # subtract elements from another counter + >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch + 0 + >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch + -1 + + ''' + if hasattr(iterable, 'iteritems'): + for elem, count in iterable.iteritems(): + self[elem] -= count + else: + for elem in iterable: + self[elem] -= 1 + + def copy(self): + 'Return a shallow copy.' + return self.__class__(self) + + def __reduce__(self): + return self.__class__, (dict(self), ) + + def __delitem__(self, elem): + 'Like dict.__delitem__() but does not raise KeyError for missing values.' + if elem in self: + dict.__delitem__(self, elem) + + def __repr__(self): + if not self: + return '%s()' % self.__class__.__name__ + items = ', '.join(map('%r: %r'.__mod__, self.most_common())) + return '%s({%s})' % (self.__class__.__name__, items) + + # Multiset-style mathematical operations discussed in: + # Knuth TAOCP Volume II section 4.6.3 exercise 19 + # and at http://en.wikipedia.org/wiki/Multiset + # + # Outputs guaranteed to only include positive counts. + # + # To strip negative and zero counts, add-in an empty counter: + # c += Counter() + + def __add__(self, other): + '''Add counts from two counters. + + >>> Counter('abbb') + Counter('bcc') + Counter({'b': 4, 'c': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem in set(self) | set(other): + newcount = self[elem] + other[elem] + if newcount > 0: + result[elem] = newcount + return result + + def __sub__(self, other): + ''' Subtract count, but keep only results with positive counts. + + >>> Counter('abbbc') - Counter('bccd') + Counter({'b': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem in set(self) | set(other): + newcount = self[elem] - other[elem] + if newcount > 0: + result[elem] = newcount + return result + + def __or__(self, other): + '''Union is the maximum of value in either of the input counters. + + >>> Counter('abbb') | Counter('bcc') + Counter({'b': 3, 'c': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + _max = max + result = Counter() + for elem in set(self) | set(other): + newcount = _max(self[elem], other[elem]) + if newcount > 0: + result[elem] = newcount + return result + + def __and__(self, other): + ''' Intersection is the minimum of corresponding counts. + + >>> Counter('abbb') & Counter('bcc') + Counter({'b': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + _min = min + result = Counter() + if len(self) < len(other): + self, other = other, self + for elem in ifilter(self.__contains__, other): + newcount = _min(self[elem], other[elem]) + if newcount > 0: + result[elem] = newcount + return result + + if __name__ == '__main__': + import doctest + print(doctest.testmod()) diff --git a/giscanner/collections/ordereddict.py b/giscanner/collections/ordereddict.py new file mode 100644 index 00000000..0cb4b956 --- /dev/null +++ b/giscanner/collections/ordereddict.py @@ -0,0 +1,120 @@ +# -*- Mode: Python -*- +# GObject-Introspection - a framework for introspecting GObject libraries +# Copyright (C) 2008 Johan Dahlin +# Copyright (C) 2013 Dieter Verfaillie <dieterv@optionexplicit.be> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + + +# Borrowed from: +# http://hg.sqlalchemy.org/sqlalchemy/raw-file/77e2264283d4/lib/sqlalchemy/util/_collections.py +# http://hg.sqlalchemy.org/sqlalchemy/raw-file/77e2264283d4/AUTHORS +# +# util/_collections.py +# Copyright (C) 2005-2012 the SQLAlchemy authors and contributors <see AUTHORS file> +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +class OrderedDict(dict): + """A dict that returns keys/values/items in the order they were added.""" + + def __init__(self, ____sequence=None, **kwargs): + self._list = [] + if ____sequence is None: + if kwargs: + self.update(**kwargs) + else: + self.update(____sequence, **kwargs) + + def clear(self): + self._list = [] + dict.clear(self) + + def copy(self): + return self.__copy__() + + def __copy__(self): + return OrderedDict(self) + + def sort(self, *arg, **kw): + self._list.sort(*arg, **kw) + + def update(self, ____sequence=None, **kwargs): + if ____sequence is not None: + if hasattr(____sequence, 'keys'): + for key in ____sequence.keys(): + self.__setitem__(key, ____sequence[key]) + else: + for key, value in ____sequence: + self[key] = value + if kwargs: + self.update(kwargs) + + def setdefault(self, key, value): + if key not in self: + self.__setitem__(key, value) + return value + else: + return self.__getitem__(key) + + def __iter__(self): + return iter(self._list) + + def values(self): + return [self[key] for key in self._list] + + def itervalues(self): + return iter([self[key] for key in self._list]) + + def keys(self): + return list(self._list) + + def iterkeys(self): + return iter(self.keys()) + + def items(self): + return [(key, self[key]) for key in self.keys()] + + def iteritems(self): + return iter(self.items()) + + def __setitem__(self, key, obj): + if key not in self: + try: + self._list.append(key) + except AttributeError: + # work around Python pickle loads() with + # dict subclass (seems to ignore __setstate__?) + self._list = [key] + dict.__setitem__(self, key, obj) + + def __delitem__(self, key): + dict.__delitem__(self, key) + self._list.remove(key) + + def pop(self, key, *default): + present = key in self + value = dict.pop(self, key, *default) + if present: + self._list.remove(key) + return value + + def popitem(self): + item = dict.popitem(self) + self._list.remove(item[0]) + return item diff --git a/giscanner/docmain.py b/giscanner/docmain.py index 8089a6b3..e65b57a0 100644 --- a/giscanner/docmain.py +++ b/giscanner/docmain.py @@ -21,9 +21,11 @@ import os import optparse -from .mallardwriter import MallardWriter +from .docwriter import DocWriter +from .sectionparser import generate_sections_file, write_sections_file from .transformer import Transformer + def doc_main(args): parser = optparse.OptionParser('%prog [options] GIR-file') @@ -32,14 +34,18 @@ def doc_main(args): help="Directory to write output to") parser.add_option("-l", "--language", action="store", dest="language", - default="Python", + default="c", help="Output language") + parser.add_option("", "--add-include-path", + action="append", dest="include_paths", default=[], + help="include paths for other GIR files") + parser.add_option("", "--write-sections-file", + action="store_true", dest="write_sections", + help="Generate and write out a sections file") options, args = parser.parse_args(args) if not options.output: raise SystemExit("missing output parameter") - if not os.path.isdir(options.output): - raise SystemExit("wrong output parameter: %s" % (options.output, )) if len(args) < 2: raise SystemExit("Need an input GIR filename") @@ -50,9 +56,17 @@ def doc_main(args): extra_include_dirs = [os.path.join(top_srcdir, 'gir'), top_builddir] else: extra_include_dirs = [] + extra_include_dirs.extend(options.include_paths) transformer = Transformer.parse_from_gir(args[1], extra_include_dirs) - writer = MallardWriter(transformer, options.language) - writer.write(options.output) + if options.write_sections: + sections_file = generate_sections_file(transformer) + + fp = open(options.output, 'w') + write_sections_file(fp, sections_file) + fp.close() + else: + writer = DocWriter(transformer, options.language) + writer.write(options.output) return 0 diff --git a/giscanner/doctemplates/C/class.tmpl b/giscanner/doctemplates/C/class.tmpl new file mode 100644 index 00000000..3f18b021 --- /dev/null +++ b/giscanner/doctemplates/C/class.tmpl @@ -0,0 +1,2 @@ +<%! page_type="guide" %>\ +<%inherit file="/class.tmpl"/> diff --git a/giscanner/doctemplates/C/constructor.tmpl b/giscanner/doctemplates/C/constructor.tmpl new file mode 100644 index 00000000..a03d2825 --- /dev/null +++ b/giscanner/doctemplates/C/constructor.tmpl @@ -0,0 +1 @@ +<%inherit file="./function.tmpl"/> diff --git a/giscanner/doctemplates/C/default.tmpl b/giscanner/doctemplates/C/default.tmpl new file mode 100644 index 00000000..b66ae926 --- /dev/null +++ b/giscanner/doctemplates/C/default.tmpl @@ -0,0 +1 @@ +<%inherit file="/base.tmpl"/> diff --git a/giscanner/doctemplates/C/enum.tmpl b/giscanner/doctemplates/C/enum.tmpl new file mode 100644 index 00000000..1523e0df --- /dev/null +++ b/giscanner/doctemplates/C/enum.tmpl @@ -0,0 +1,2 @@ +<%! page_type="guide" %>\ +<%inherit file="/base.tmpl"/> diff --git a/giscanner/doctemplates/C/function.tmpl b/giscanner/doctemplates/C/function.tmpl new file mode 100644 index 00000000..8d669438 --- /dev/null +++ b/giscanner/doctemplates/C/function.tmpl @@ -0,0 +1,61 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <api:function> + <api:returns> + <api:type>${formatter.format_type(node.retval.type) | x}</api:type> + </api:returns> + <api:name>${formatter.format_function_name(node)}</api:name> +% for arg in formatter.get_parameters(node): +% if arg.type.ctype == '<varargs>': + <api:varargs/> +% else: + <api:arg> + <api:type>${formatter.format_type(arg.type) | x}</api:type> + <api:name>${formatter.format_parameter_name(node, arg)}</api:name> + </api:arg> +% endif +% endfor + </api:function> +</%block> +<%block name="synopsis"> +<synopsis><code mime="text/x-csrc"> +${node.retval.type.ctype} ${formatter.format_function_name(node)} (\ +% if not formatter.get_parameters(node): +void\ +% else: +% for ix, arg in enumerate(formatter.get_parameters(node)): +% if ix != 0: +${' ' * (len(formatter.format_type(node.retval.type)) + len(formatter.format_function_name(node)) + 3)}\ +% endif +% if arg.type.ctype == '<varargs>': +...\ +% else: +${formatter.format_type(arg.type) | x} ${arg.argname}\ +% endif +% if ix != len(formatter.get_parameters(node)) - 1: +, +% endif +% endfor +% endif +); +</code></synopsis> +</%block> +<%block name="details"> +% if formatter.get_parameters(node) or node.retval: +<terms> +% for arg in formatter.get_parameters(node): +<item> +<title><code>${arg.argname}</code></title> +${formatter.format(node, arg.doc)} +</item> +% endfor +% if node.retval: +<item> +<title><code>Returns</code></title> +${formatter.format(node, node.retval.doc)} +</item> +% endif +</terms> +% endif +</%block> diff --git a/giscanner/doctemplates/C/method.tmpl b/giscanner/doctemplates/C/method.tmpl new file mode 100644 index 00000000..a03d2825 --- /dev/null +++ b/giscanner/doctemplates/C/method.tmpl @@ -0,0 +1 @@ +<%inherit file="./function.tmpl"/> diff --git a/giscanner/doctemplates/C/namespace.tmpl b/giscanner/doctemplates/C/namespace.tmpl new file mode 100644 index 00000000..cb8195da --- /dev/null +++ b/giscanner/doctemplates/C/namespace.tmpl @@ -0,0 +1 @@ +<%inherit file="/namespace.tmpl"/> diff --git a/giscanner/doctemplates/C/property.tmpl b/giscanner/doctemplates/C/property.tmpl new file mode 100644 index 00000000..6ec9e8e1 --- /dev/null +++ b/giscanner/doctemplates/C/property.tmpl @@ -0,0 +1,5 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <title type="link" role="topic">${node.name}</title> +</%block> diff --git a/giscanner/doctemplates/C/record.tmpl b/giscanner/doctemplates/C/record.tmpl new file mode 100644 index 00000000..b66ae926 --- /dev/null +++ b/giscanner/doctemplates/C/record.tmpl @@ -0,0 +1 @@ +<%inherit file="/base.tmpl"/> diff --git a/giscanner/doctemplates/C/signal.tmpl b/giscanner/doctemplates/C/signal.tmpl new file mode 100644 index 00000000..28c0b740 --- /dev/null +++ b/giscanner/doctemplates/C/signal.tmpl @@ -0,0 +1,5 @@ +<%inherit file="./function.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <title type="link" role="topic">${node.name}</title> +</%block> diff --git a/giscanner/doctemplates/C/vfunc.tmpl b/giscanner/doctemplates/C/vfunc.tmpl new file mode 100644 index 00000000..aa0394f4 --- /dev/null +++ b/giscanner/doctemplates/C/vfunc.tmpl @@ -0,0 +1,4 @@ +<%inherit file="./function.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} +</%block> diff --git a/giscanner/doctemplates/Gjs/class.tmpl b/giscanner/doctemplates/Gjs/class.tmpl new file mode 100644 index 00000000..887c646b --- /dev/null +++ b/giscanner/doctemplates/Gjs/class.tmpl @@ -0,0 +1,18 @@ +<%inherit file="/class.tmpl"/> +<%block name="synopsis"> + <synopsis><code> +const ${namespace.name} = imports.gi.${namespace.name}; + +let ${formatter.to_underscores(node.name).lower()} = new ${namespace.name}.${node.name}(\ +% if len(node.properties) > 0: +{ +% for ix, property_ in enumerate(node.properties): +% if property_.construct or property_.construct_only or property_.writable: + <link xref='${namespace.name}.${node.name}-${property_.name}'>${property_.name.replace('-', '_')}</link>: value, +% endif +% endfor +}\ +% endif +); + </code></synopsis> +</%block> diff --git a/giscanner/doctemplates/Gjs/constructor.tmpl b/giscanner/doctemplates/Gjs/constructor.tmpl new file mode 100644 index 00000000..a03d2825 --- /dev/null +++ b/giscanner/doctemplates/Gjs/constructor.tmpl @@ -0,0 +1 @@ +<%inherit file="./function.tmpl"/> diff --git a/giscanner/doctemplates/Gjs/default.tmpl b/giscanner/doctemplates/Gjs/default.tmpl new file mode 100644 index 00000000..b66ae926 --- /dev/null +++ b/giscanner/doctemplates/Gjs/default.tmpl @@ -0,0 +1 @@ +<%inherit file="/base.tmpl"/> diff --git a/giscanner/doctemplates/Gjs/enum.tmpl b/giscanner/doctemplates/Gjs/enum.tmpl new file mode 100644 index 00000000..35cdd439 --- /dev/null +++ b/giscanner/doctemplates/Gjs/enum.tmpl @@ -0,0 +1,13 @@ +<%inherit file="/base.tmpl"/> +<%block name="details"> +% if node.members: +<terms> +% for member in node.members: +<item> +<title><code>${node.name}.${member.name.upper()}</code></title> +${formatter.format(node, member.doc)} +</item> +% endfor +</terms> +% endif +</%block> diff --git a/giscanner/doctemplates/Gjs/function.tmpl b/giscanner/doctemplates/Gjs/function.tmpl new file mode 100644 index 00000000..e0fd9612 --- /dev/null +++ b/giscanner/doctemplates/Gjs/function.tmpl @@ -0,0 +1,48 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <api:function> + <api:returns> + <api:type>${formatter.format_type(node.retval.type) | x}</api:type> + </api:returns> + <api:name>${node.symbol}</api:name> +% for arg in formatter.get_parameters(node): +% if arg.type.ctype == '<varargs>': + <api:varargs/> +% else: + <api:arg> + <api:type>${formatter.format_type(arg.type) | x}</api:type> + <api:name>${formatter.format_parameter_name(node, arg)}</api:name> + </api:arg> +% endif +% endfor + </api:function> +</%block> +<%block name="synopsis"> +<synopsis><code mime="text/x-gjs"> +function \ +${node.name}(\ +${', '.join('%s:%s' % (arg.argname, formatter.format_type(arg.type)) for arg in formatter.get_parameters(node))}\ +):${formatter.format_type(node.retval.type)} { + // Gjs wrapper for ${node.symbol}() +} +</code></synopsis> +</%block> +<%block name="details"> +% if formatter.get_parameters(node) or node.retval: +<terms> +% for arg in formatter.get_parameters(node): +<item> +<title><code>${arg.argname}</code></title> +${formatter.format(node, arg.doc)} +</item> +% endfor +% if node.retval and node.retval.type.ctype != 'void': +<item> +<title><code>Returns</code></title> +${formatter.format(node, node.retval.doc)} +</item> +% endif +</terms> +% endif +</%block> diff --git a/giscanner/doctemplates/Gjs/method.tmpl b/giscanner/doctemplates/Gjs/method.tmpl new file mode 100644 index 00000000..a03d2825 --- /dev/null +++ b/giscanner/doctemplates/Gjs/method.tmpl @@ -0,0 +1 @@ +<%inherit file="./function.tmpl"/> diff --git a/giscanner/doctemplates/Gjs/namespace.tmpl b/giscanner/doctemplates/Gjs/namespace.tmpl new file mode 100644 index 00000000..4d80c2a5 --- /dev/null +++ b/giscanner/doctemplates/Gjs/namespace.tmpl @@ -0,0 +1,2 @@ +<%! page_type="guide" %>\ +<%inherit file="/namespace.tmpl"/> diff --git a/giscanner/doctemplates/Gjs/property.tmpl b/giscanner/doctemplates/Gjs/property.tmpl new file mode 100644 index 00000000..3316a00c --- /dev/null +++ b/giscanner/doctemplates/Gjs/property.tmpl @@ -0,0 +1,10 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <title type="link" role="topic">${node.name}</title> +</%block> +<%block name="synopsis"> +<synopsis><code mime="text/x-python"> +"${node.name}" ${formatter.format_type(node.type)} : ${formatter.format_property_flags(node)} +</code></synopsis> +</%block> diff --git a/giscanner/doctemplates/Gjs/record.tmpl b/giscanner/doctemplates/Gjs/record.tmpl new file mode 100644 index 00000000..1523e0df --- /dev/null +++ b/giscanner/doctemplates/Gjs/record.tmpl @@ -0,0 +1,2 @@ +<%! page_type="guide" %>\ +<%inherit file="/base.tmpl"/> diff --git a/giscanner/doctemplates/Gjs/signal.tmpl b/giscanner/doctemplates/Gjs/signal.tmpl new file mode 100644 index 00000000..084d9743 --- /dev/null +++ b/giscanner/doctemplates/Gjs/signal.tmpl @@ -0,0 +1,37 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <title type="link" role="topic">${node.name}</title> +</%block> +<%block name="synopsis"> +<synopsis><code mime="text/x-python"> +function callback(${formatter.to_underscores(node.parent.name).lower()}, \ +% for arg in formatter.get_parameters(node): +${arg.argname}:${formatter.format_type(arg.type)}, \ +% endfor +):${formatter.format_type(node.retval.type)}; +</code></synopsis> +</%block> +<%block name="details"> +<terms> +<item> +<title><code>${formatter.to_underscores(node.parent.name).lower()}</code></title> +<p>instance of ${formatter.format_xref(node.parent)} that is emitting the signal</p> +</item> +% for arg in formatter.get_parameters(node): +<item> +<title><code>${arg.argname}</code></title> +${formatter.format(node, arg.doc)} +</item> +% endfor +% if node.retval and \ + node.retval.type.ctype != 'void' and \ + node.retval.type.ctype is not None: +<item> +<title><code>Returns</code></title> +${formatter.format(node, node.retval.doc)} +</item> +% endif +</terms> +</%block> + diff --git a/giscanner/doctemplates/Gjs/vfunc.tmpl b/giscanner/doctemplates/Gjs/vfunc.tmpl new file mode 100644 index 00000000..1cbe511c --- /dev/null +++ b/giscanner/doctemplates/Gjs/vfunc.tmpl @@ -0,0 +1,27 @@ +<%inherit file="/base.tmpl"/> +<%block name="synopsis"> +<synopsis><code mime="text/x-gjs"> +function vfunc_${node.name}(\ +${', '.join('%s:%s' % (arg.argname, formatter.format_type(arg.type)) for arg in formatter.get_parameters(node))}\ +):${formatter.format_type(node.retval.type)} { +} +</code></synopsis> +</%block> +<%block name="details"> +% if formatter.get_parameters(node) or node.retval: +<terms> +% for arg in formatter.get_parameters(node): +<item> +<title><code>${arg.argname}</code></title> +${formatter.format(node, arg.doc)} +</item> +% endfor +% if node.retval and node.retval.type.ctype != 'void': +<item> +<title><code>Returns</code></title> +${formatter.format(node, node.retval.doc)} +</item> +% endif +</terms> +% endif +</%block> diff --git a/giscanner/doctemplates/Python/class.tmpl b/giscanner/doctemplates/Python/class.tmpl new file mode 100644 index 00000000..435b31a5 --- /dev/null +++ b/giscanner/doctemplates/Python/class.tmpl @@ -0,0 +1,17 @@ +<%inherit file="/class.tmpl"/> +<%block name="synopsis"> + <synopsis><code> +from gi.repository import ${namespace.name} + +${formatter.to_underscores(node.name).lower()} = ${namespace.name}.${node.name}(\ +% for ix, property_ in enumerate(node.properties): +% if property_.construct or property_.construct_only or property_.writable: +<link xref='${namespace.name}.${node.name}-${property_.name}'>${property_.name.replace('-', '_')}</link>=value\ +% if ix != len(node.properties) - 1: +, \ +% endif +% endif +% endfor +)\ + </code></synopsis> +</%block> diff --git a/giscanner/doctemplates/Python/constructor.tmpl b/giscanner/doctemplates/Python/constructor.tmpl new file mode 100644 index 00000000..a03d2825 --- /dev/null +++ b/giscanner/doctemplates/Python/constructor.tmpl @@ -0,0 +1 @@ +<%inherit file="./function.tmpl"/> diff --git a/giscanner/doctemplates/Python/default.tmpl b/giscanner/doctemplates/Python/default.tmpl new file mode 100644 index 00000000..b66ae926 --- /dev/null +++ b/giscanner/doctemplates/Python/default.tmpl @@ -0,0 +1 @@ +<%inherit file="/base.tmpl"/> diff --git a/giscanner/doctemplates/Python/enum.tmpl b/giscanner/doctemplates/Python/enum.tmpl new file mode 100644 index 00000000..35cdd439 --- /dev/null +++ b/giscanner/doctemplates/Python/enum.tmpl @@ -0,0 +1,13 @@ +<%inherit file="/base.tmpl"/> +<%block name="details"> +% if node.members: +<terms> +% for member in node.members: +<item> +<title><code>${node.name}.${member.name.upper()}</code></title> +${formatter.format(node, member.doc)} +</item> +% endfor +</terms> +% endif +</%block> diff --git a/giscanner/doctemplates/Python/function.tmpl b/giscanner/doctemplates/Python/function.tmpl new file mode 100644 index 00000000..072a1185 --- /dev/null +++ b/giscanner/doctemplates/Python/function.tmpl @@ -0,0 +1,53 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <api:function> + <api:returns> + <api:type>${formatter.format_type(node.retval.type) | x}</api:type> + </api:returns> + <api:name>${node.symbol}</api:name> +% for arg in formatter.get_parameters(node): +% if arg.type.ctype == '<varargs>': + <api:varargs/> +% else: + <api:arg> + <api:type>${formatter.format_type(arg.type) | x}</api:type> + <api:name>${formatter.format_parameter_name(node, arg)}</api:name> + </api:arg> +% endif +% endfor + </api:function> +</%block> +<%block name="synopsis"> +<synopsis><code mime="text/x-python"> +% if formatter.get_parameters(node): +@accepts(\ +${', '.join((formatter.format_type(arg.type) for arg in formatter.get_parameters(node)))}\ +) +% endif +@returns(${formatter.format_type(node.retval.type) | x}) +def \ +${node.name}(\ +${', '.join((formatter.format_parameter_name(node, arg) for arg in formatter.get_parameters(node)))}\ +): + # Python wrapper for ${node.symbol}() +</code></synopsis> +</%block> +<%block name="details"> +% if formatter.get_parameters(node) or node.retval: +<terms> +% for ix, arg in enumerate(formatter.get_parameters(node)): +<item> +<title><code>${formatter.format_parameter_name(node, arg)}</code></title> +${formatter.format(node, arg.doc)} +</item> +% endfor +% if node.retval and node.retval.type.ctype != 'void': +<item> +<title><code>Returns</code></title> +{formatter.format(node, node.retval.doc)} +</item> +% endif +</terms> +% endif +</%block> diff --git a/giscanner/doctemplates/Python/method.tmpl b/giscanner/doctemplates/Python/method.tmpl new file mode 100644 index 00000000..a03d2825 --- /dev/null +++ b/giscanner/doctemplates/Python/method.tmpl @@ -0,0 +1 @@ +<%inherit file="./function.tmpl"/> diff --git a/giscanner/doctemplates/Python/namespace.tmpl b/giscanner/doctemplates/Python/namespace.tmpl new file mode 100644 index 00000000..4d80c2a5 --- /dev/null +++ b/giscanner/doctemplates/Python/namespace.tmpl @@ -0,0 +1,2 @@ +<%! page_type="guide" %>\ +<%inherit file="/namespace.tmpl"/> diff --git a/giscanner/doctemplates/Python/property.tmpl b/giscanner/doctemplates/Python/property.tmpl new file mode 100644 index 00000000..3316a00c --- /dev/null +++ b/giscanner/doctemplates/Python/property.tmpl @@ -0,0 +1,10 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <title type="link" role="topic">${node.name}</title> +</%block> +<%block name="synopsis"> +<synopsis><code mime="text/x-python"> +"${node.name}" ${formatter.format_type(node.type)} : ${formatter.format_property_flags(node)} +</code></synopsis> +</%block> diff --git a/giscanner/doctemplates/Python/record.tmpl b/giscanner/doctemplates/Python/record.tmpl new file mode 100644 index 00000000..1523e0df --- /dev/null +++ b/giscanner/doctemplates/Python/record.tmpl @@ -0,0 +1,2 @@ +<%! page_type="guide" %>\ +<%inherit file="/base.tmpl"/> diff --git a/giscanner/doctemplates/Python/signal.tmpl b/giscanner/doctemplates/Python/signal.tmpl new file mode 100644 index 00000000..dc931107 --- /dev/null +++ b/giscanner/doctemplates/Python/signal.tmpl @@ -0,0 +1,42 @@ +<%inherit file="/base.tmpl"/> +<%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + <title type="link" role="topic">${node.name}</title> +</%block> +<%block name="synopsis"> +<synopsis><code mime="text/x-python"> +def callback(${formatter.to_underscores(node.parent.name).lower()}, \ +% for arg in formatter.get_parameters(node): +${arg.argname}, \ +% endfor +user_param1, ...) +</code></synopsis> +</%block> +<%block name="details"> +<terms> +<item> +<title><code>${formatter.to_underscores(node.parent.name).lower()}</code></title> +<p>instance of ${formatter.format_xref(node.parent)} that is emitting the signal</p> +</item> +% for arg in formatter.get_parameters(node): +<item> +<title><code>${arg.argname}</code></title> +${formatter.format(node, arg.doc)} +</item> +% endfor +<title><code>user_param1</code></title> +<p>first user parameter (if any) specified with the connect() method</p> +<item> +<title><code>...</code></title> +<p>additional user parameters (if any)</p> +</item> +% if node.retval and \ + node.retval.type.ctype != 'void' and \ + node.retval.type.ctype is not None: +<item> +<title><code>Returns</code></title> +${formatter.format(node, node.retval.doc)} +</item> +% endif +</terms> +</%block> diff --git a/giscanner/doctemplates/Python/vfunc.tmpl b/giscanner/doctemplates/Python/vfunc.tmpl new file mode 100644 index 00000000..98a30932 --- /dev/null +++ b/giscanner/doctemplates/Python/vfunc.tmpl @@ -0,0 +1,33 @@ +<%inherit file="/base.tmpl"/> +<%block name="synopsis"> +<synopsis><code mime="text/x-python"> +% if formatter.get_parameters(node): +@accepts(\ +${', '.join((formatter.format_type(arg.type) for arg in formatter.get_parameters(node)))}\ +) +% endif +@returns(${formatter.format_type(node.retval.type) | x}) +def \ +do_${node.name}(\ +${', '.join((arg.argname for arg in formatter.get_parameters(node)))}\ +): +</code></synopsis> +</%block> +<%block name="details"> +% if formatter.get_parameters(node) or node.retval: +<terms> +% for arg in formatter.get_parameters(node): +<item> +<title><code>${arg.argname}</code></title> +${formatter.format(node, arg.doc)} +</item> +% endfor +% if node.retval and node.retval.type.ctype != 'void': +<item> +<title><code>Returns</code></title> +${formatter.format(node, node.retval.doc)} +</item> +% endif +</terms> +% endif +</%block> diff --git a/giscanner/doctemplates/base.tmpl b/giscanner/doctemplates/base.tmpl new file mode 100644 index 00000000..78980773 --- /dev/null +++ b/giscanner/doctemplates/base.tmpl @@ -0,0 +1,29 @@ +<%! page_type="topic" %>\ +<?xml version="1.0"?> +<page id="${page_id}" + type="${self.attr.page_type}" + style="${page_kind}" + xmlns="http://projectmallard.org/1.0/" + xmlns:api="http://projectmallard.org/experimental/api/" + xmlns:ui="http://projectmallard.org/1.0/ui/"> + <info> + <%block name="info"> + ${formatter.format_xref(node.parent, type="guide", group=page_kind)} + </%block> + </info> + <title><%block name="title">${formatter.format_page_name(node)}</%block></title> + <%block name="synopsis"> + </%block> + <%block name="doc"> + ${formatter.format(node, node.doc)} + </%block> + <%block name="since_version"> + % if node.version: + <p>Since ${node.version}</p> + % endif + </%block> + <%block name="details"> + </%block> + <%block name="links"> + </%block> +</page> diff --git a/giscanner/doctemplates/class.tmpl b/giscanner/doctemplates/class.tmpl new file mode 100644 index 00000000..7f8b6869 --- /dev/null +++ b/giscanner/doctemplates/class.tmpl @@ -0,0 +1,40 @@ +<%! page_type="guide" %>\ +<%inherit file="/base.tmpl"/> +<%block name="details"> + <synopsis> + <title>Hierarchy</title> + <tree> +% for class_ in formatter.get_class_hierarchy(node): + <item> + <code>${class_.namespace.name}.${class_.name}</code> +% endfor +% for class_ in formatter.get_class_hierarchy(node): + </item> +% endfor + </tree> + </synopsis> +</%block> +<%block name="links"> + <links type="topic" ui:expanded="true" + api:type="function" api:mime="${formatter.mime_type}" + groups="method" style="linklist"> + <title>Methods</title> + </links> + <links type="topic" ui:expanded="true" + api:type="function" api:mime="${formatter.mime_type}" + groups="function" style="linklist"> + <title>Functions</title> + </links> + <links type="topic" ui:expanded="true" groups="property" style="linklist"> + <title>Properties</title> + </links> + <links type="topic" ui:expanded="true" groups="signal" style="linklist"> + <title>Signals</title> + </links> + <links type="topic" ui:expanded="true" groups="vfunc" style="linklist"> + <title>Virtual functions</title> + </links> + <links type="topic" ui:expanded="true" groups="#first #default #last" style="linklist"> + <title>Other</title> + </links> +</%block> diff --git a/giscanner/doctemplates/namespace.tmpl b/giscanner/doctemplates/namespace.tmpl new file mode 100644 index 00000000..bb58bb16 --- /dev/null +++ b/giscanner/doctemplates/namespace.tmpl @@ -0,0 +1,19 @@ +<%! page_type="guide" %>\ +<%inherit file="/base.tmpl"/> +<%block name="doc"> +</%block> +<%block name="info"> +</%block> +<%block name="links"> + <links type="topic" ui:expanded="true" groups="class" style="linklist"> + <title>Classes</title> + </links> + <links type="topic" ui:expanded="true" groups="function" style="linklist"> + <title>Functions</title> + </links> + <links type="topic" ui:expanded="true" groups="#first #default #last" style="linklist"> + <title>Other</title> + </links> +</%block> +<%block name="since_version"> +</%block> diff --git a/giscanner/docwriter.py b/giscanner/docwriter.py new file mode 100644 index 00000000..a4c817e9 --- /dev/null +++ b/giscanner/docwriter.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python +# -*- Mode: Python -*- +# GObject-Introspection - a framework for introspecting GObject libraries +# Copyright (C) 2010 Zach Goldberg +# Copyright (C) 2011 Johan Dahlin +# Copyright (C) 2011 Shaun McCance +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + +import os +import re +import tempfile + +from xml.sax import saxutils +from mako.lookup import TemplateLookup + +from . import ast, xmlwriter +from .utils import to_underscores + + +def make_page_id(node, recursive=False): + if isinstance(node, ast.Namespace): + if recursive: + return node.name + else: + return 'index' + + if hasattr(node, '_chain') and node._chain: + parent = node._chain[-1] + else: + parent = None + + if parent is None: + return '%s.%s' % (node.namespace.name, node.name) + + if isinstance(node, (ast.Property, ast.Signal, ast.VFunction)): + return '%s-%s' % (make_page_id(parent, recursive=True), node.name) + else: + return '%s.%s' % (make_page_id(parent, recursive=True), node.name) + + +def get_node_kind(node): + if isinstance(node, ast.Namespace): + node_kind = 'namespace' + elif isinstance(node, (ast.Class, ast.Interface)): + node_kind = 'class' + elif isinstance(node, ast.Record): + node_kind = 'record' + elif isinstance(node, ast.Function): + if node.is_method: + node_kind = 'method' + elif node.is_constructor: + node_kind = 'constructor' + else: + node_kind = 'function' + elif isinstance(node, ast.Enum): + node_kind = 'enum' + elif isinstance(node, ast.Property) and node.parent is not None: + node_kind = 'property' + elif isinstance(node, ast.Signal) and node.parent is not None: + node_kind = 'signal' + elif isinstance(node, ast.VFunction) and node.parent is not None: + node_kind = 'vfunc' + else: + node_kind = 'default' + + return node_kind + + +class TemplatedScanner(object): + def __init__(self, specs): + self.specs = self.unmangle_specs(specs) + self.regex = self.make_regex(self.specs) + + def unmangle_specs(self, specs): + mangled = re.compile('<<([a-zA-Z_:]+)>>') + specdict = dict((name.lstrip('!'), spec) for name, spec in specs) + + def unmangle(spec, name=None): + def replace_func(match): + child_spec_name = match.group(1) + + if ':' in child_spec_name: + pattern_name, child_spec_name = child_spec_name.split(':', 1) + else: + pattern_name = None + + child_spec = specdict[child_spec_name] + # Force all child specs of this one to be unnamed + unmangled = unmangle(child_spec, None) + if pattern_name and name: + return '(?P<%s_%s>%s)' % (name, pattern_name, unmangled) + else: + return unmangled + + return mangled.sub(replace_func, spec) + + return [(name, unmangle(spec, name)) for name, spec in specs] + + def make_regex(self, specs): + regex = '|'.join('(?P<%s>%s)' % (name, spec) for name, spec in specs + if not name.startswith('!')) + return re.compile(regex) + + def get_properties(self, name, match): + groupdict = match.groupdict() + properties = {name: groupdict.pop(name)} + name = name + "_" + for group, value in groupdict.iteritems(): + if group.startswith(name): + key = group[len(name):] + properties[key] = value + return properties + + def scan(self, text): + pos = 0 + while True: + match = self.regex.search(text, pos) + if match is None: + break + + start = match.start() + if start > pos: + yield ('other', text[pos:start], None) + + pos = match.end() + name = match.lastgroup + yield (name, match.group(0), self.get_properties(name, match)) + + if pos < len(text): + yield ('other', text[pos:], None) + + +class DocstringScanner(TemplatedScanner): + def __init__(self): + specs = [ + ('!alpha', r'[a-zA-Z0-9_]+'), + ('!alpha_dash', r'[a-zA-Z0-9_-]+'), + ('property', r'#<<type_name:alpha>>:(<<property_name:alpha_dash>>)'), + ('signal', r'#<<type_name:alpha>>::(<<signal_name:alpha_dash>>)'), + ('type_name', r'#(<<type_name:alpha>>)'), + ('enum_value', r'%(<<member_name:alpha>>)'), + ('parameter', r'@<<param_name:alpha>>'), + ('function_call', r'<<symbol_name:alpha>>\(\)'), + ] + + super(DocstringScanner, self).__init__(specs) + + +class DocFormatter(object): + def __init__(self, transformer): + self._transformer = transformer + self._scanner = DocstringScanner() + + def escape(self, text): + return saxutils.escape(text) + + def should_render_node(self, node): + if isinstance(node, ast.Constant): + return False + + if getattr(node, "private", False): + return False + + return True + + def format(self, node, doc): + if doc is None: + return '' + + result = '' + for para in doc.split('\n\n'): + result += '<p>' + result += self.format_inline(node, para) + result += '</p>' + return result + + def _resolve_type(self, ident): + try: + matches = self._transformer.split_ctype_namespaces(ident) + except ValueError: + return None + for namespace, name in matches: + node = namespace.get(name) + if node: + return node + return None + + def _resolve_symbol(self, symbol): + try: + matches = self._transformer.split_csymbol_namespaces(symbol) + except ValueError: + return None + for namespace, name in matches: + node = namespace.get_by_symbol(symbol) + if node: + return node + return None + + def _find_thing(self, list_, name): + for item in list_: + if item.name == name: + return item + raise KeyError("Could not find %s" % (name, )) + + def _process_other(self, node, match, props): + return self.escape(match) + + def _process_property(self, node, match, props): + type_node = self._resolve_type(props['type_name']) + if type_node is None: + return match + + try: + prop = self._find_thing(type_node.properties, props['property_name']) + except (AttributeError, KeyError): + return match + + return self.format_xref(prop) + + def _process_signal(self, node, match, props): + type_node = self._resolve_type(props['type_name']) + if type_node is None: + return match + + try: + signal = self._find_thing(type_node.signals, props['signal_name']) + except (AttributeError, KeyError): + return match + + return self.format_xref(signal) + + def _process_type_name(self, node, match, props): + type_ = self._resolve_type(props['type_name']) + if type_ is None: + return match + + return self.format_xref(type_) + + def _process_enum_value(self, node, match, props): + member_name = props['member_name'] + + try: + return '<code>%s</code>' % (self.fundamentals[member_name], ) + except KeyError: + pass + + enum_value = self._resolve_symbol(member_name) + if enum_value: + return self.format_xref(enum_value) + + return match + + def _process_parameter(self, node, match, props): + try: + parameter = node.get_parameter(props['param_name']) + except (AttributeError, ValueError): + return match + + return '<code>%s</code>' % (self.format_parameter_name(node, parameter), ) + + def _process_function_call(self, node, match, props): + func = self._resolve_symbol(props['symbol_name']) + if func is None: + return match + + return self.format_xref(func) + + def _process_token(self, node, tok): + kind, match, props = tok + + dispatch = { + 'other': self._process_other, + 'property': self._process_property, + 'signal': self._process_signal, + 'type_name': self._process_type_name, + 'enum_value': self._process_enum_value, + 'parameter': self._process_parameter, + 'function_call': self._process_function_call, + } + + return dispatch[kind](node, match, props) + + def get_parameters(self, node): + raise NotImplementedError + + def format_inline(self, node, para): + tokens = self._scanner.scan(para) + words = [self._process_token(node, tok) for tok in tokens] + return ''.join(words) + + def format_parameter_name(self, node, parameter): + if isinstance(parameter.type, ast.Varargs): + return "..." + else: + return parameter.argname + + def format_function_name(self, func): + raise NotImplementedError + + def format_type(self, type_): + raise NotImplementedError + + def format_page_name(self, node): + if isinstance(node, ast.Namespace): + return 'Index' + elif isinstance(node, ast.Function): + return self.format_function_name(node) + elif isinstance(node, ast.Property) and node.parent is not None: + return '%s:%s' % (self.format_page_name(node.parent), node.name) + elif isinstance(node, ast.Signal) and node.parent is not None: + return '%s::%s' % (self.format_page_name(node.parent), node.name) + elif isinstance(node, ast.VFunction) and node.parent is not None: + return '%s::%s' % (self.format_page_name(node.parent), node.name) + else: + return make_page_id(node) + + def format_xref(self, node, **attrdict): + if node is None: + attrs = [('xref', 'index')] + attrdict.items() + return xmlwriter.build_xml_tag('link', attrs) + elif isinstance(node, ast.Member): + # Enum/BitField members are linked to the main enum page. + return self.format_xref(node.parent, **attrdict) + '.' + node.name + else: + attrs = [('xref', make_page_id(node))] + attrdict.items() + return xmlwriter.build_xml_tag('link', attrs) + + def format_property_flags(self, property_, construct_only=False): + flags = [] + if property_.readable and not construct_only: + flags.append("Read") + if property_.writable and not construct_only: + flags.append("Write") + if property_.construct: + flags.append("Construct") + if property_.construct_only: + flags.append("Construct Only") + + return " / ".join(flags) + + def to_underscores(self, string): + return to_underscores(string) + + def get_class_hierarchy(self, node): + parent_chain = [node] + + while node.parent_type: + node = self._transformer.lookup_typenode(node.parent_type) + parent_chain.append(node) + + parent_chain.reverse() + return parent_chain + + +class DocFormatterC(DocFormatter): + language = "C" + mime_type = "text/x-csrc" + + fundamentals = { + "TRUE": "TRUE", + "FALSE": "FALSE", + "NULL": "NULL", + } + + def format_type(self, type_): + if isinstance(type_, ast.Array): + return self.format_type(type_.element_type) + '*' + elif type_.ctype is not None: + return type_.ctype + elif type_.target_fundamental: + return type_.target_fundamental + else: + node = self._transformer.lookup_typenode(type_) + return getattr(node, 'ctype') + + def format_function_name(self, func): + if isinstance(func, ast.Function): + return func.symbol + else: + return func.name + + def get_parameters(self, node): + return node.all_parameters + + +class DocFormatterIntrospectableBase(DocFormatter): + def should_render_node(self, node): + if isinstance(node, ast.Record) and node.is_gtype_struct_for is not None: + return False + + if not getattr(node, "introspectable", True): + return False + + return super(DocFormatterIntrospectableBase, self).should_render_node(node) + + +class DocFormatterPython(DocFormatterIntrospectableBase): + language = "Python" + mime_type = "text/python" + + fundamentals = { + "TRUE": "True", + "FALSE": "False", + "NULL": "None", + } + + def should_render_node(self, node): + if getattr(node, "is_constructor", False): + return False + + return super(DocFormatterPython, self).should_render_node(node) + + def is_method(self, node): + if getattr(node, "is_method", False): + return True + + if isinstance(node, ast.VFunction): + return True + + return False + + def format_parameter_name(self, node, parameter): + # Force "self" for the first parameter of a method + if self.is_method(node) and parameter is node.instance_parameter: + return "self" + elif isinstance(parameter.type, ast.Varargs): + return "..." + else: + return parameter.argname + + def format_fundamental_type(self, name): + fundamental_types = { + "utf8": "unicode", + "gunichar": "unicode", + "gchar": "str", + "guchar": "str", + "gboolean": "bool", + "gint": "int", + "guint": "int", + "glong": "int", + "gulong": "int", + "gint64": "int", + "guint64": "int", + "gfloat": "float", + "gdouble": "float", + "gchararray": "str", + "GParam": "GLib.Param", + "PyObject": "object", + "GStrv": "[str]", + "GVariant": "GLib.Variant"} + + return fundamental_types.get(name, name) + + def format_type(self, type_): + if isinstance(type_, (ast.List, ast.Array)): + return '[' + self.format_type(type_.element_type) + ']' + elif isinstance(type_, ast.Map): + return '{%s: %s}' % (self.format_type(type_.key_type), + self.format_type(type_.value_type)) + elif type_.target_giname is not None: + return type_.target_giname + else: + return self.format_fundamental_type(type_.target_fundamental) + + def format_function_name(self, func): + if func.parent is not None: + return "%s.%s" % (self.format_page_name(func.parent), func.name) + else: + return func.name + + def get_parameters(self, node): + return node.all_parameters + + +class DocFormatterGjs(DocFormatterIntrospectableBase): + language = "Gjs" + mime_type = "text/x-gjs" + + fundamentals = { + "TRUE": "true", + "FALSE": "false", + "NULL": "null", + } + + def is_method(self, node): + if getattr(node, "is_method", False): + return True + + if isinstance(node, ast.VFunction): + return True + + return False + + def format_fundamental_type(self, name): + fundamental_types = { + "utf8": "String", + "gunichar": "String", + "gchar": "String", + "guchar": "String", + "gboolean": "Boolean", + "gint": "Number", + "guint": "Number", + "glong": "Number", + "gulong": "Number", + "gint64": "Number", + "guint64": "Number", + "gfloat": "Number", + "gdouble": "Number", + "gchararray": "String", + "GParam": "GLib.Param", + "PyObject": "Object", + "GStrv": "[String]", + "GVariant": "GLib.Variant"} + + return fundamental_types.get(name, name) + + def format_type(self, type_): + if isinstance(type_, (ast.List, ast.Array)): + return '[' + self.format_type(type_.element_type) + ']' + elif isinstance(type_, ast.Map): + return '{%s: %s}' % (self.format_type(type_.key_type), + self.format_type(type_.value_type)) + elif type_.target_fundamental == "none": + return "void" + elif type_.target_giname is not None: + return type_.target_giname + else: + return self.format_fundamental_type(type_.target_fundamental) + + def format_function_name(self, func): + if func.is_method: + return "%s.prototype.%s" % (self.format_page_name(func.parent), func.name) + elif func.is_constructor: + return "%s.%s" % (self.format_page_name(func.parent), func.name) + else: + return func.name + + def get_parameters(self, node): + skip = [] + for param in node.parameters: + if param.direction == ast.PARAM_DIRECTION_OUT: + skip.append(param) + if param.closure_name is not None: + skip.append(node.get_parameter(param.closure_name)) + if param.destroy_name is not None: + skip.append(node.get_parameter(param.destroy_name)) + if isinstance(param.type, ast.Array) and param.type.length_param_name is not None: + skip.append(node.get_parameter(param.type.length_param_name)) + + params = [] + for param in node.parameters: + if param not in skip: + params.append(param) + return params + + +LANGUAGES = { + "c": DocFormatterC, + "python": DocFormatterPython, + "gjs": DocFormatterGjs, +} + + +class DocWriter(object): + def __init__(self, transformer, language): + self._transformer = transformer + + try: + formatter_class = LANGUAGES[language.lower()] + except KeyError: + raise SystemExit("Unsupported language: %s" % (language, )) + + self._formatter = formatter_class(self._transformer) + self._language = self._formatter.language + + self._lookup = self._get_template_lookup() + + def _get_template_lookup(self): + if 'UNINSTALLED_INTROSPECTION_SRCDIR' in os.environ: + top_srcdir = os.environ['UNINSTALLED_INTROSPECTION_SRCDIR'] + srcdir = os.path.join(top_srcdir, 'giscanner') + else: + srcdir = os.path.dirname(__file__) + + template_dir = os.path.join(srcdir, 'doctemplates') + + return TemplateLookup(directories=[template_dir], + module_directory=tempfile.mkdtemp(), + output_encoding='utf-8') + + def write(self, output): + try: + os.makedirs(output) + except OSError: + # directory already made + pass + + self._walk_node(output, self._transformer.namespace, []) + self._transformer.namespace.walk(lambda node, chain: self._walk_node(output, node, chain)) + + def _walk_node(self, output, node, chain): + if isinstance(node, ast.Function) and node.moved_to is not None: + return False + if getattr(node, 'disguised', False): + return False + if self._formatter.should_render_node(node): + self._render_node(node, chain, output) + return True + return False + + def _render_node(self, node, chain, output): + namespace = self._transformer.namespace + + # A bit of a hack...maybe this should be an official API + node._chain = list(chain) + + page_kind = get_node_kind(node) + template_name = '%s/%s.tmpl' % (self._language, page_kind) + page_id = make_page_id(node) + + template = self._lookup.get_template(template_name) + result = template.render(namespace=namespace, + node=node, + page_id=page_id, + page_kind=page_kind, + formatter=self._formatter) + + output_file_name = os.path.join(os.path.abspath(output), + page_id + '.page') + fp = open(output_file_name, 'w') + fp.write(result) + fp.close() diff --git a/giscanner/dumper.py b/giscanner/dumper.py index f78d2ae8..b415dd13 100644 --- a/giscanner/dumper.py +++ b/giscanner/dumper.py @@ -45,7 +45,9 @@ main(int argc, char **argv) GError *error = NULL; const char *introspect_dump_prefix = "--introspect-dump="; +#if !GLIB_CHECK_VERSION(2,35,0) g_type_init (); +#endif %(init_sections)s @@ -80,10 +82,16 @@ class DumpCompiler(object): self._get_type_functions = get_type_functions self._error_quark_functions = error_quark_functions - self._compiler_cmd = os.environ.get('CC', 'gcc') + self._compiler_cmd = os.environ.get('CC', 'cc') self._linker_cmd = os.environ.get('CC', self._compiler_cmd) self._pkgconfig_cmd = os.environ.get('PKG_CONFIG', 'pkg-config') - + self._pkgconfig_msvc_flags = '' + # Enable the --msvc-syntax pkg-config flag when + # the Microsoft compiler is used + # (This is the other way to check whether Visual C++ is used subsequently) + args = self._compiler_cmd.split() + if 'cl.exe' in args or 'cl' in args: + self._pkgconfig_msvc_flags = '--msvc-syntax' self._uninst_srcdir = os.environ.get( 'UNINSTALLED_INTROSPECTION_SRCDIR') self._packages = ['gio-2.0 gmodule-2.0'] @@ -143,7 +151,12 @@ class DumpCompiler(object): f.write("\n};\n") f.close() - o_path = self._generate_tempfile(tmpdir, '.o') + # Microsoft compilers generate intermediate .obj files + # during compilation, unlike .o files like GCC and others + if self._pkgconfig_msvc_flags: + o_path = self._generate_tempfile(tmpdir, '.obj') + else: + o_path = self._generate_tempfile(tmpdir, '.o') if os.name == 'nt': ext = 'exe' @@ -154,14 +167,14 @@ class DumpCompiler(object): try: self._compile(o_path, c_path) - except CompilerError, e: + except CompilerError as e: if not utils.have_debug_flag('save-temps'): shutil.rmtree(tmpdir) raise SystemExit('compilation of temporary binary failed:' + str(e)) try: self._link(bin_path, o_path) - except LinkerError, e: + except LinkerError as e: if not utils.have_debug_flag('save-temps'): shutil.rmtree(tmpdir) raise SystemExit('linking of temporary binary failed: ' + str(e)) @@ -176,8 +189,14 @@ class DumpCompiler(object): return os.path.join(tmpdir, tmpl) def _run_pkgconfig(self, flag): + # Enable the --msvc-syntax pkg-config flag when + # the Microsoft compiler is used + if self._pkgconfig_msvc_flags: + cmd = [self._pkgconfig_cmd, self._pkgconfig_msvc_flags, flag] + else: + cmd = [self._pkgconfig_cmd, flag] proc = subprocess.Popen( - [self._pkgconfig_cmd, flag] + self._packages, + cmd + self._packages, stdout=subprocess.PIPE) return proc.communicate()[0].split() @@ -188,14 +207,28 @@ class DumpCompiler(object): # header of the library being introspected if self._compiler_cmd == 'gcc' and not self._options.init_sections: args.append('-Wall') + # The Microsoft compiler uses different option flags for + # silencing warnings on deprecated function usage + if self._pkgconfig_msvc_flags: + args.append("-wd4996") + else: + args.append("-Wno-deprecated-declarations") pkgconfig_flags = self._run_pkgconfig('--cflags') - args.extend(pkgconfig_flags) + args.extend([utils.cflag_real_include_path(f) for f in pkgconfig_flags]) + cppflags = os.environ.get('CPPFLAGS', '') + for cppflag in cppflags.split(): + args.append(cppflag) cflags = os.environ.get('CFLAGS', '') for cflag in cflags.split(): args.append(cflag) for include in self._options.cpp_includes: args.append('-I' + include) - args.extend(['-c', '-o', output]) + # The Microsoft compiler uses different option flags for + # compilation result output + if self._pkgconfig_msvc_flags: + args.extend(['-c', '-Fe' + output, '-Fo' + output]) + else: + args.extend(['-c', '-o', output]) for source in sources: if not os.path.exists(source): raise CompilerError( @@ -207,7 +240,7 @@ class DumpCompiler(object): sys.stdout.flush() try: subprocess.check_call(args) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: raise CompilerError(e) def _link(self, output, *sources): @@ -221,13 +254,21 @@ class DumpCompiler(object): args.append('--silent') args.extend(self._linker_cmd.split()) - args.extend(['-o', output]) + # We can use -o for the Microsoft compiler/linker, + # but it is considered deprecated usage with that + if self._pkgconfig_msvc_flags: + args.extend(['-Fe' + output]) + else: + args.extend(['-o', output]) if libtool: if os.name == 'nt': args.append('-export-all-symbols') else: args.append('-export-dynamic') + cppflags = os.environ.get('CPPFLAGS', '') + for cppflag in cppflags.split(): + args.append(cppflag) cflags = os.environ.get('CFLAGS', '') for cflag in cflags.split(): args.append(cflag) @@ -256,7 +297,7 @@ class DumpCompiler(object): sys.stdout.flush() try: subprocess.check_call(args) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: raise LinkerError(e) def _add_link_internal_args(self, args, libtool): @@ -264,26 +305,52 @@ class DumpCompiler(object): # is being built in the current directory. # Search the current directory first - args.append('-L.') + # (This flag is not supported nor needed for Visual C++) + if self._pkgconfig_msvc_flags == '': + args.append('-L.') # https://bugzilla.gnome.org/show_bug.cgi?id=625195 if not libtool: - args.append('-Wl,-rpath=.') + # We don't have -Wl,-rpath for Visual C++, and that's + # going to cause a problem. Instead, link to internal + # libraries by deducing the .lib file name using + # the namespace name and version + if self._pkgconfig_msvc_flags: + if self._options.namespace_version: + args.append(str.lower(self._options.namespace_name) + + '-' + + self._options.namespace_version + '.lib') + else: + args.append(str.lower(self._options.namespace_name) + '.lib') + else: + args.append('-Wl,-rpath=.') + + # Ensure libraries are always linked as we are going to use ldd to work + # out their names later + if not libtool and self._pkgconfig_msvc_flags == '': + args.append('-Wl,--no-as-needed') for library in self._options.libraries: - if library.endswith(".la"): # explicitly specified libtool library - args.append(library) - else: - args.append('-l' + library) + # Visual C++: We have the needed .lib files now, and we need to link + # to .lib files, not the .dll as the --library option specifies the + # .dll(s) the .gir file refers to + if self._pkgconfig_msvc_flags == '': + if library.endswith(".la"): # explicitly specified libtool library + args.append(library) + else: + args.append('-l' + library) for library_path in self._options.library_paths: - args.append('-L' + library_path) - if os.path.isabs(library_path): - if libtool: - args.append('-rpath') - args.append(library_path) - else: - args.append('-Wl,-rpath=' + library_path) + # Not used/needed on Visual C++, and -Wl,-rpath options + # will cause grief + if self._pkgconfig_msvc_flags == '': + args.append('-L' + library_path) + if os.path.isabs(library_path): + if libtool: + args.append('-rpath') + args.append(library_path) + else: + args.append('-Wl,-rpath=' + library_path) args.extend(self._run_pkgconfig('--libs')) @@ -294,10 +361,14 @@ class DumpCompiler(object): args.extend(self._run_pkgconfig('--libs')) for library in self._options.libraries: - if library.endswith(".la"): # explicitly specified libtool library - args.append(library) - else: - args.append('-l' + library) + # The --library option on Windows pass in the .dll file(s) the + # .gir files refer to, so don't link to them on Visual C++ + if self._pkgconfig_msvc_flags == '': + if library.endswith(".la"): # explicitly specified libtool library + args.append(library) + else: + args.append('-l' + library) + def compile_introspection_binary(options, get_type_functions, error_quark_functions): diff --git a/giscanner/gdumpparser.py b/giscanner/gdumpparser.py index c0b13f4a..e1fc9358 100644 --- a/giscanner/gdumpparser.py +++ b/giscanner/gdumpparser.py @@ -165,7 +165,7 @@ blob containing data gleaned from GObject's primitive introspection.""" try: try: subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: # Clean up temporaries raise SystemExit(e) return parse(out_path) @@ -203,8 +203,7 @@ blob containing data gleaned from GObject's primitive introspection.""" def _initparse_gobject_record(self, record): if (record.name.startswith('ParamSpec') - and not record.name in ('ParamSpecPool', 'ParamSpecClass', - 'ParamSpecTypeInfo')): + and not record.name in ('ParamSpecPool', 'ParamSpecClass', 'ParamSpecTypeInfo')): parent = None if record.name != 'ParamSpec': parent = ast.Type(target_giname='GObject.ParamSpec') @@ -251,7 +250,7 @@ blob containing data gleaned from GObject's primitive introspection.""" (get_type, c_symbol_prefix) = self._split_type_and_symbol_prefix(xmlnode) try: enum_name = self._transformer.strip_identifier(type_name) - except TransformerException, e: + except TransformerException as e: message.fatal(e) # The scanned member values are more accurate than the values that the @@ -280,7 +279,6 @@ blob containing data gleaned from GObject's primitive introspection.""" member.attrib['name'], member.attrib['nick'])) - if xmlnode.tag == 'flags': klass = ast.Bitfield else: @@ -299,7 +297,7 @@ blob containing data gleaned from GObject's primitive introspection.""" (ns, name) = self._transformer.split_csymbol(get_type) assert ns is self._namespace if name in ('get_type', '_get_gtype'): - message.fatal("""The GObject name %r isn't compatibile + message.fatal("""The GObject name %r isn't compatible with the configured identifier prefixes: %r The class would have no name. Most likely you want to specify a @@ -316,7 +314,7 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide (get_type, c_symbol_prefix) = self._split_type_and_symbol_prefix(xmlnode) try: object_name = self._transformer.strip_identifier(type_name) - except TransformerException, e: + except TransformerException as e: message.fatal(e) node = ast.Class(object_name, None, gtype_name=type_name, @@ -328,17 +326,6 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide self._introspect_signals(node, xmlnode) self._introspect_implemented_interfaces(node, xmlnode) self._add_record_fields(node) - - if node.name == 'InitiallyUnowned': - # http://bugzilla.gnome.org/show_bug.cgi?id=569408 - # GInitiallyUnowned is actually a typedef for GObject, but - # that's not reflected in the GIR, where it appears as a - # subclass (as it appears in the GType hierarchy). So - # what we do here is copy all of the GObject fields into - # GInitiallyUnowned so that struct offset computation - # works correctly. - node.fields = self._namespace.get('Object').fields - self._namespace.append(node, replace=True) def _introspect_interface(self, xmlnode): @@ -346,7 +333,7 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide (get_type, c_symbol_prefix) = self._split_type_and_symbol_prefix(xmlnode) try: interface_name = self._transformer.strip_identifier(type_name) - except TransformerException, e: + except TransformerException as e: message.fatal(e) node = ast.Interface(interface_name, None, gtype_name=type_name, @@ -391,7 +378,7 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide try: name = self._transformer.strip_identifier(type_name) - except TransformerException, e: + except TransformerException as e: message.fatal(e) # This one doesn't go in the main namespace; we associate it with # the struct or union @@ -437,7 +424,7 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide if i == 0: argname = 'object' else: - argname = 'p%s' % (i-1, ) + argname = 'p%s' % (i - 1, ) pctype = parameter.attrib['type'] ptype = ast.Type.create_from_gtype_name(pctype) param = ast.Parameter(argname, ptype) @@ -465,7 +452,7 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide (get_type, c_symbol_prefix) = self._split_type_and_symbol_prefix(xmlnode) try: fundamental_name = self._transformer.strip_identifier(type_name) - except TransformerException, e: + except TransformerException as e: message.warn(e) return @@ -509,7 +496,7 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide def _pair_boxed_type(self, boxed): try: name = self._transformer.strip_identifier(boxed.gtype_name) - except TransformerException, e: + except TransformerException as e: message.fatal(e) pair_node = self._namespace.get(name) if not pair_node: @@ -525,15 +512,6 @@ different --identifier-prefix.""" % (xmlnode.attrib['name'], self._namespace.ide else: return False - def _strip_class_suffix(self, name): - if (name.endswith('Class') or - name.endswith('Iface')): - return name[:-5] - elif name.endswith('Interface'): - return name[:-9] - else: - return None - def _find_class_record(self, cls): pair_record = None if isinstance(cls, ast.Class): diff --git a/giscanner/girparser.py b/giscanner/girparser.py index eb53a3c4..25e9035d 100644 --- a/giscanner/girparser.py +++ b/giscanner/girparser.py @@ -46,9 +46,6 @@ class GIRParser(object): def __init__(self, types_only=False): self._types_only = types_only - self._shared_libraries = [] - self._includes = set() - self._pkgconfig_packages = set() self._namespace = None self._filename_stack = [] @@ -62,10 +59,9 @@ class GIRParser(object): self._filename_stack.pop() def parse_tree(self, tree): - self._includes.clear() self._namespace = None - self._shared_libraries = [] self._pkgconfig_packages = set() + self._includes = set() self._c_includes = set() self._c_prefix = None self._parse_api(tree.getroot()) @@ -73,26 +69,6 @@ class GIRParser(object): def get_namespace(self): return self._namespace - def get_shared_libraries(self): - return self._shared_libraries - - def get_includes(self): - return self._includes - - def get_c_includes(self): - return self._c_includes - - def get_c_prefix(self): - return self._c_prefix - - def get_pkgconfig_packages(self): - if not hasattr(self, '_pkgconfig_packages'): - self._pkgconfig_packages = [] - return self._pkgconfig_packages - - def get_doc(self): - return parse(self._filename) - # Private def _find_first_child(self, node, name_or_names): @@ -122,9 +98,8 @@ class GIRParser(object): assert root.tag == _corens('repository') version = root.attrib['version'] if version != COMPATIBLE_GIR_VERSION: - raise SystemExit("%s: Incompatible version %s (supported: %s)" \ - % (self._get_current_file(), - version, COMPATIBLE_GIR_VERSION)) + raise SystemExit("%s: Incompatible version %s (supported: %s)" % + (self._get_current_file(), version, COMPATIBLE_GIR_VERSION)) for node in root.getchildren(): if node.tag == _corens('include'): @@ -143,12 +118,14 @@ class GIRParser(object): if symbol_prefixes: symbol_prefixes = symbol_prefixes.split(',') self._namespace = ast.Namespace(ns.attrib['name'], - ns.attrib['version'], - identifier_prefixes=identifier_prefixes, - symbol_prefixes=symbol_prefixes) + ns.attrib['version'], + identifier_prefixes=identifier_prefixes, + symbol_prefixes=symbol_prefixes) if 'shared-library' in ns.attrib: - self._shared_libraries.extend( - ns.attrib['shared-library'].split(',')) + self._namespace.shared_libraries = ns.attrib['shared-library'].split(',') + self._namespace.includes = self._includes + self._namespace.c_includes = self._c_includes + self._namespace.exported_packages = self._pkgconfig_packages parser_methods = { _corens('alias'): self._parse_alias, @@ -159,8 +136,7 @@ class GIRParser(object): _corens('interface'): self._parse_object_interface, _corens('record'): self._parse_record, _corens('union'): self._parse_union, - _glibns('boxed'): self._parse_boxed, - } + _glibns('boxed'): self._parse_boxed} if not self._types_only: parser_methods[_corens('constant')] = self._parse_constant @@ -172,8 +148,7 @@ class GIRParser(object): method(node) def _parse_include(self, node): - include = ast.Include(node.attrib['name'], - node.attrib['version']) + include = ast.Include(node.attrib['name'], node.attrib['version']) self._includes.add(include) def _parse_pkgconfig_package(self, node): @@ -184,9 +159,7 @@ class GIRParser(object): def _parse_alias(self, node): typeval = self._parse_type(node) - alias = ast.Alias(node.attrib['name'], - typeval, - node.attrib.get(_cns('type'))) + alias = ast.Alias(node.attrib['name'], typeval, node.attrib.get(_cns('type'))) self._parse_generic_attribs(node, alias) self._namespace.append(alias) @@ -204,12 +177,24 @@ class GIRParser(object): version = node.attrib.get('version') if version: obj.version = version - deprecated = node.attrib.get('deprecated') + version_doc = node.find(_corens('doc-version')) + if version_doc is not None: + if version_doc.text: + obj.version_doc = version_doc.text + deprecated = node.attrib.get('deprecated-version') if deprecated: obj.deprecated = deprecated - deprecated_version = node.attrib.get('deprecated-version') - if deprecated_version: - obj.deprecated_version = deprecated_version + deprecated_doc = node.find(_corens('doc-deprecated')) + if deprecated_doc is not None: + if deprecated_doc.text: + obj.deprecated_doc = deprecated_doc.text + stability = node.attrib.get('stability') + if stability: + obj.stability = stability + stability_doc = node.find(_corens('doc-stability')) + if stability_doc is not None: + if stability_doc.text: + obj.stability_doc = stability_doc.text def _parse_object_interface(self, node): parent = node.attrib.get('parent') @@ -218,9 +203,9 @@ class GIRParser(object): else: parent_type = None - ctor_args = [node.attrib['name'], - parent_type] - ctor_kwargs = {'gtype_name': node.attrib[_glibns('type-name')], + ctor_kwargs = {'name': node.attrib['name'], + 'parent_type': parent_type, + 'gtype_name': node.attrib[_glibns('type-name')], 'get_type': node.attrib[_glibns('get-type')], 'c_symbol_prefix': node.attrib.get(_cns('symbol-prefix')), 'ctype': node.attrib.get(_cns('type'))} @@ -234,7 +219,7 @@ class GIRParser(object): else: raise AssertionError(node) - obj = klass(*ctor_args, **ctor_kwargs) + obj = klass(**ctor_kwargs) self._parse_generic_attribs(node, obj) type_struct = node.attrib.get(_glibns('type-struct')) if type_struct: @@ -247,10 +232,11 @@ class GIRParser(object): 'set-value-func', 'get-value-func']: func_name = node.attrib.get(_glibns(func_id)) obj.__dict__[func_id.replace('-', '_')] = func_name - self._namespace.append(obj) if self._types_only: + self._namespace.append(obj) return + for iface in self._find_children(node, _corens('implements')): obj.interfaces.append(self._namespace.type_from_name(iface.attrib['name'])) for iface in self._find_children(node, _corens('prerequisite')): @@ -272,12 +258,14 @@ class GIRParser(object): func = self._parse_function_common(ctor, ast.Function, obj) func.is_constructor = True obj.constructors.append(func) - obj.fields.extend(self._parse_fields(node)) + obj.fields.extend(self._parse_fields(node, obj)) for prop in self._find_children(node, _corens('property')): obj.properties.append(self._parse_property(prop, obj)) for signal in self._find_children(node, _glibns('signal')): obj.signals.append(self._parse_function_common(signal, ast.Signal, obj)) + self._namespace.append(obj) + def _parse_callback(self, node): callback = self._parse_function_common(node, ast.Callback) self._namespace.append(callback) @@ -286,6 +274,18 @@ class GIRParser(object): function = self._parse_function_common(node, ast.Function) self._namespace.append(function) + def _parse_parameter(self, node): + typeval = self._parse_type(node) + param = ast.Parameter(node.attrib.get('name'), + typeval, + node.attrib.get('direction') or ast.PARAM_DIRECTION_IN, + node.attrib.get('transfer-ownership'), + node.attrib.get('allow-none') == '1', + node.attrib.get('scope'), + node.attrib.get('caller-allocates') == '1') + self._parse_generic_attribs(node, param) + return param + def _parse_function_common(self, node, klass, parent=None): name = node.attrib['name'] returnnode = node.find(_corens('return-value')) @@ -323,21 +323,15 @@ class GIRParser(object): parameters_node = node.find(_corens('parameters')) if (parameters_node is not None): + paramnode = self._find_first_child(parameters_node, _corens('instance-parameter')) + if paramnode: + func.instance_parameter = self._parse_parameter(paramnode) for paramnode in self._find_children(parameters_node, _corens('parameter')): - typeval = self._parse_type(paramnode) - param = ast.Parameter(paramnode.attrib.get('name'), - typeval, - paramnode.attrib.get('direction') or ast.PARAM_DIRECTION_IN, - paramnode.attrib.get('transfer-ownership'), - paramnode.attrib.get('allow-none') == '1', - paramnode.attrib.get('scope'), - paramnode.attrib.get('caller-allocates') == '1') - self._parse_generic_attribs(paramnode, param) - parameters.append(param) + parameters.append(self._parse_parameter(paramnode)) for i, paramnode in enumerate(self._find_children(parameters_node, _corens('parameter'))): param = parameters[i] - self._parse_type_second_pass(func, paramnode, param.type) + self._parse_type_array_length(parameters, paramnode, param.type) closure = paramnode.attrib.get('closure') if closure: idx = int(closure) @@ -349,19 +343,19 @@ class GIRParser(object): assert idx < len(parameters), "%d >= %d" % (idx, len(parameters)) param.destroy_name = parameters[idx].argname - self._parse_type_second_pass(func, returnnode, retval.type) + self._parse_type_array_length(parameters, returnnode, retval.type) self._parse_generic_attribs(node, func) self._namespace.track(func) return func - def _parse_fields(self, node): + def _parse_fields(self, node, obj): res = [] names = (_corens('field'), _corens('record'), _corens('union'), _corens('callback')) for child in node.getchildren(): if child.tag in names: - fieldobj = self._parse_field(child) + fieldobj = self._parse_field(child, obj) res.append(fieldobj) return res @@ -376,16 +370,21 @@ class GIRParser(object): compound.foreign = True self._parse_generic_attribs(node, compound) if not self._types_only: - compound.fields.extend(self._parse_fields(node)) + compound.fields.extend(self._parse_fields(node, compound)) for method in self._find_children(node, _corens('method')): - compound.methods.append( - self._parse_function_common(method, ast.Function, compound)) + func = self._parse_function_common(method, ast.Function, compound) + func.is_method = True + compound.methods.append(func) + for i, fieldnode in enumerate(self._find_children(node, _corens('field'))): + field = compound.fields[i] + self._parse_type_array_length(compound.fields, fieldnode, field.type) for func in self._find_children(node, _corens('function')): compound.static_methods.append( self._parse_function_common(func, ast.Function, compound)) for ctor in self._find_children(node, _corens('constructor')): - compound.constructors.append( - self._parse_function_common(ctor, ast.Function, compound)) + func = self._parse_function_common(ctor, ast.Function, compound) + func.is_constructor = True + compound.constructors.append(func) return compound def _parse_record(self, node, anonymous=False): @@ -435,7 +434,8 @@ class GIRParser(object): return ast.Type(ctype=ctype) elif name in ['GLib.List', 'GLib.SList']: subchild = self._find_first_child(typenode, - map(_corens, ('callback', 'array', 'varargs', 'type'))) + map(_corens, ('callback', 'array', + 'varargs', 'type'))) if subchild is not None: element_type = self._parse_type(typenode) else: @@ -446,9 +446,7 @@ class GIRParser(object): subchildren_types = map(self._parse_type_simple, subchildren) while len(subchildren_types) < 2: subchildren_types.append(ast.TYPE_ANY) - return ast.Map(subchildren_types[0], - subchildren_types[1], - ctype=ctype) + return ast.Map(subchildren_types[0], subchildren_types[1], ctype=ctype) else: return self._namespace.type_from_name(name, ctype) else: @@ -461,8 +459,8 @@ class GIRParser(object): return self._parse_type_simple(typenode) assert False, "Failed to parse toplevel type" - def _parse_type_second_pass(self, parent, node, typeval): - """A hack necessary to handle the integer parameter indexes on + def _parse_type_array_length(self, siblings, node, typeval): + """A hack necessary to handle the integer parameter/field indexes on array types.""" typenode = node.find(_corens('array')) if typenode is None: @@ -470,9 +468,11 @@ class GIRParser(object): lenidx = typenode.attrib.get('length') if lenidx is not None: idx = int(lenidx) - assert idx < len(parent.parameters), "%r %d >= %d" \ - % (parent, idx, len(parent.parameters)) - typeval.length_param_name = parent.parameters[idx].argname + assert idx < len(siblings), "%r %d >= %d" % (parent, idx, len(siblings)) + if isinstance(siblings[idx], ast.Field): + typeval.length_param_name = siblings[idx].name + else: + typeval.length_param_name = siblings[idx].argname def _parse_boxed(self, node): obj = ast.Boxed(node.attrib[_glibns('name')], @@ -480,9 +480,11 @@ class GIRParser(object): get_type=node.attrib[_glibns('get-type')], c_symbol_prefix=node.attrib.get(_cns('symbol-prefix'))) self._parse_generic_attribs(node, obj) - self._namespace.append(obj) + if self._types_only: + self._namespace.append(obj) return + for method in self._find_children(node, _corens('method')): func = self._parse_function_common(method, ast.Function, obj) func.is_method = True @@ -493,8 +495,9 @@ class GIRParser(object): for callback in self._find_children(node, _corens('callback')): obj.fields.append( self._parse_function_common(callback, ast.Callback, obj)) + self._namespace.append(obj) - def _parse_field(self, node): + def _parse_field(self, node, parent): type_node = None anonymous_node = None if node.tag in map(_corens, ('record', 'union')): @@ -514,23 +517,24 @@ class GIRParser(object): assert node.tag == _corens('field'), node.tag type_node = self._parse_type(node) field = ast.Field(node.attrib.get('name'), - type_node, - node.attrib.get('readable') != '0', - node.attrib.get('writable') == '1', - node.attrib.get('bits'), - anonymous_node=anonymous_node) + type_node, + node.attrib.get('readable') != '0', + node.attrib.get('writable') == '1', + node.attrib.get('bits'), + anonymous_node=anonymous_node) field.private = node.attrib.get('private') == '1' + field.parent = parent self._parse_generic_attribs(node, field) return field def _parse_property(self, node, parent): prop = ast.Property(node.attrib['name'], - self._parse_type(node), - node.attrib.get('readable') != '0', - node.attrib.get('writable') == '1', - node.attrib.get('construct') == '1', - node.attrib.get('construct-only') == '1', - node.attrib.get('transfer-ownership')) + self._parse_type(node), + node.attrib.get('readable') != '0', + node.attrib.get('writable') == '1', + node.attrib.get('construct') == '1', + node.attrib.get('construct-only') == '1', + node.attrib.get('transfer-ownership')) self._parse_generic_attribs(node, prop) prop.parent = parent return prop @@ -570,12 +574,16 @@ class GIRParser(object): obj.error_domain = glib_error_domain obj.ctype = ctype self._parse_generic_attribs(node, obj) - self._namespace.append(obj) if self._types_only: + self._namespace.append(obj) return - for member in self._find_children(node, _corens('member')): - members.append(self._parse_member(member)) + + for member_node in self._find_children(node, _corens('member')): + member = self._parse_member(member_node) + member.parent = obj + members.append(member) for func_node in self._find_children(node, _corens('function')): func = self._parse_function_common(func_node, ast.Function) obj.static_methods.append(func) + self._namespace.append(obj) diff --git a/giscanner/girwriter.py b/giscanner/girwriter.py index 97f81616..304cf322 100644 --- a/giscanner/girwriter.py +++ b/giscanner/girwriter.py @@ -28,40 +28,32 @@ from .xmlwriter import XMLWriter # Compatible changes we just make inline COMPATIBLE_GIR_VERSION = '1.2' + class GIRWriter(XMLWriter): - def __init__(self, namespace, shlibs, includes, pkgs, c_includes): + def __init__(self, namespace): super(GIRWriter, self).__init__() self.write_comment( -'''This file was automatically generated from C sources - DO NOT EDIT! -To affect the contents of this file, edit the original C definitions, -and/or use gtk-doc annotations. ''') - self._write_repository(namespace, shlibs, includes, pkgs, - c_includes) - - def _write_repository(self, namespace, shlibs, includes=None, - packages=None, c_includes=None): - if includes is None: - includes = frozenset() - if packages is None: - packages = frozenset() - if c_includes is None: - c_includes = frozenset() + 'This file was automatically generated from C sources - DO NOT EDIT!\n' + 'To affect the contents of this file, edit the original C definitions,\n' + 'and/or use gtk-doc annotations. ') + self._write_repository(namespace) + + def _write_repository(self, namespace): attrs = [ ('version', COMPATIBLE_GIR_VERSION), ('xmlns', 'http://www.gtk.org/introspection/core/1.0'), ('xmlns:c', 'http://www.gtk.org/introspection/c/1.0'), - ('xmlns:glib', 'http://www.gtk.org/introspection/glib/1.0'), - ] + ('xmlns:glib', 'http://www.gtk.org/introspection/glib/1.0')] with self.tagcontext('repository', attrs): - for include in sorted(includes): + for include in sorted(namespace.includes): self._write_include(include) - for pkg in sorted(set(packages)): + for pkg in sorted(set(namespace.exported_packages)): self._write_pkgconfig_pkg(pkg) - for c_include in sorted(set(c_includes)): + for c_include in sorted(set(namespace.c_includes)): self._write_c_include(c_include) self._namespace = namespace - self._write_namespace(namespace, shlibs) + self._write_namespace(namespace) self._namespace = None def _write_include(self, include): @@ -76,10 +68,10 @@ and/or use gtk-doc annotations. ''') attrs = [('name', c_include)] self.write_tag('c:include', attrs) - def _write_namespace(self, namespace, shlibs): + def _write_namespace(self, namespace): attrs = [('name', namespace.name), ('version', namespace.version), - ('shared-library', ','.join(shlibs)), + ('shared-library', ','.join(namespace.shared_libraries)), ('c:identifier-prefixes', ','.join(namespace.identifier_prefixes)), ('c:symbol-prefixes', ','.join(namespace.symbol_prefixes))] with self.tagcontext('namespace', attrs): @@ -131,20 +123,39 @@ and/or use gtk-doc annotations. ''') attrs.append(('version', node.version)) def _write_generic(self, node): - for key, value in node.attributes: + for key, value in node.attributes.items(): self.write_tag('attribute', [('name', key), ('value', value)]) + if hasattr(node, 'doc') and node.doc: - self.write_tag('doc', [('xml:whitespace', 'preserve')], + self.write_tag('doc', [('xml:space', 'preserve')], node.doc) + if hasattr(node, 'version_doc') and node.version_doc: + self.write_tag('doc-version', [('xml:space', 'preserve')], + node.version_doc) + + if hasattr(node, 'deprecated_doc') and node.deprecated_doc: + self.write_tag('doc-deprecated', [('xml:space', 'preserve')], + node.deprecated_doc) + + if hasattr(node, 'stability_doc') and node.stability_doc: + self.write_tag('doc-stability', [('xml:space', 'preserve')], + node.stability_doc) + def _append_node_generic(self, node, attrs): if node.skip or not node.introspectable: attrs.append(('introspectable', '0')) + + if node.deprecated or node.deprecated_doc: + # The deprecated attribute used to contain node.deprecated_doc as an attribute. As + # an xml attribute cannot preserve whitespace, deprecated_doc has been moved into + # it's own tag, written in _write_generic() above. We continue to write the deprecated + # attribute for backwards compatibility + attrs.append(('deprecated', '1')) + if node.deprecated: - attrs.append(('deprecated', node.deprecated)) - if node.deprecated_version: - attrs.append(('deprecated-version', - node.deprecated_version)) + attrs.append(('deprecated-version', node.deprecated)) + if node.stability: attrs.append(('stability', node.stability)) @@ -170,9 +181,11 @@ and/or use gtk-doc annotations. ''') with self.tagcontext(tag_name, attrs): self._write_generic(callable) self._write_return_type(callable.retval, parent=callable) - self._write_parameters(callable, callable.parameters) + self._write_parameters(callable) def _write_function(self, func, tag_name='function'): + if func.internal_skipped: + return attrs = [] if hasattr(func, 'symbol'): attrs.append(('c:identifier', func.symbol)) @@ -204,16 +217,18 @@ and/or use gtk-doc annotations. ''') attrs.append(('skip', '1')) with self.tagcontext('return-value', attrs): self._write_generic(return_) - self._write_type(return_.type, function=parent) + self._write_type(return_.type, parent=parent) - def _write_parameters(self, parent, parameters): - if not parameters: + def _write_parameters(self, callable): + if not callable.parameters and callable.instance_parameter is None: return with self.tagcontext('parameters'): - for parameter in parameters: - self._write_parameter(parent, parameter) + if callable.instance_parameter: + self._write_parameter(callable, callable.instance_parameter, 'instance-parameter') + for parameter in callable.parameters: + self._write_parameter(callable, parameter) - def _write_parameter(self, parent, parameter): + def _write_parameter(self, parent, parameter, nodename='parameter'): attrs = [] if parameter.argname is not None: attrs.append(('name', parameter.argname)) @@ -236,9 +251,9 @@ and/or use gtk-doc annotations. ''') attrs.append(('destroy', '%d' % (idx, ))) if parameter.skip: attrs.append(('skip', '1')) - with self.tagcontext('parameter', attrs): + with self.tagcontext(nodename, attrs): self._write_generic(parameter) - self._write_type(parameter.type, function=parent) + self._write_type(parameter.type, parent=parent) def _type_to_name(self, typeval): if not typeval.resolved: @@ -271,7 +286,7 @@ and/or use gtk-doc annotations. ''') self.write_tag('type', attrs) - def _write_type(self, ntype, relation=None, function=None): + def _write_type(self, ntype, relation=None, parent=None): assert isinstance(ntype, ast.Type), ntype attrs = [] if ntype.complete_ctype: @@ -279,8 +294,7 @@ and/or use gtk-doc annotations. ''') elif ntype.ctype: attrs.append(('c:type', ntype.ctype)) if isinstance(ntype, ast.Varargs): - with self.tagcontext('varargs', []): - pass + self.write_tag('varargs', []) elif isinstance(ntype, ast.Array): if ntype.array_type != ast.Array.C: attrs.insert(0, ('name', ntype.array_type)) @@ -295,9 +309,13 @@ and/or use gtk-doc annotations. ''') if ntype.size is not None: attrs.append(('fixed-size', '%d' % (ntype.size, ))) if ntype.length_param_name is not None: - assert function - attrs.insert(0, ('length', '%d' - % (function.get_parameter_index(ntype.length_param_name, )))) + if isinstance(parent, ast.Callable): + length = parent.get_parameter_index(ntype.length_param_name) + elif isinstance(parent, ast.Compound): + length = parent.get_field_index(ntype.length_param_name) + else: + assert False, "parent not a callable or compound: %r" % parent + attrs.insert(0, ('length', '%d' % (length, ))) with self.tagcontext('array', attrs): self._write_type(ntype.element_type) @@ -363,13 +381,17 @@ and/or use gtk-doc annotations. ''') ('c:identifier', member.symbol)] if member.nick is not None: attrs.append(('glib:nick', member.nick)) - self.write_tag('member', attrs) + with self.tagcontext('member', attrs): + self._write_generic(member) def _write_constant(self, constant): attrs = [('name', constant.name), ('value', constant.value), ('c:type', constant.ctype)] + self._append_version(constant, attrs) + self._append_node_generic(constant, attrs) with self.tagcontext('constant', attrs): + self._write_generic(constant) self._write_type(constant.value_type) def _write_class(self, node): @@ -380,9 +402,9 @@ and/or use gtk-doc annotations. ''') self._append_node_generic(node, attrs) if isinstance(node, ast.Class): tag_name = 'class' - if node.parent is not None: + if node.parent_type is not None: attrs.append(('parent', - self._type_to_name(node.parent))) + self._type_to_name(node.parent_type))) if node.is_abstract: attrs.append(('abstract', '1')) else: @@ -418,9 +440,8 @@ and/or use gtk-doc annotations. ''') if isinstance(node, ast.Class): for method in sorted(node.constructors): self._write_constructor(method) - if isinstance(node, (ast.Class, ast.Interface)): - for method in sorted(node.static_methods): - self._write_static_method(method) + for method in sorted(node.static_methods): + self._write_static_method(method) for vfunc in sorted(node.virtual_methods): self._write_vfunc(vfunc) for method in sorted(node.methods): @@ -428,7 +449,7 @@ and/or use gtk-doc annotations. ''') for prop in sorted(node.properties): self._write_property(prop) for field in node.fields: - self._write_field(field) + self._write_field(field, node) for signal in sorted(node.signals): self._write_signal(signal) @@ -482,7 +503,7 @@ and/or use gtk-doc annotations. ''') attrs = list(extra_attrs) if record.name is not None: attrs.append(('name', record.name)) - if record.ctype is not None: # the record might be anonymous + if record.ctype is not None: # the record might be anonymous attrs.append(('c:type', record.ctype)) if record.disguised: attrs.append(('disguised', '1')) @@ -501,7 +522,7 @@ and/or use gtk-doc annotations. ''') self._write_generic(record) if record.fields: for field in record.fields: - self._write_field(field, is_gtype_struct) + self._write_field(field, record, is_gtype_struct) for method in sorted(record.constructors): self._write_constructor(method) for method in sorted(record.methods): @@ -513,7 +534,7 @@ and/or use gtk-doc annotations. ''') attrs = [] if union.name is not None: attrs.append(('name', union.name)) - if union.ctype is not None: # the union might be anonymous + if union.ctype is not None: # the union might be anonymous attrs.append(('c:type', union.ctype)) self._append_version(union, attrs) self._append_node_generic(union, attrs) @@ -524,7 +545,7 @@ and/or use gtk-doc annotations. ''') self._write_generic(union) if union.fields: for field in union.fields: - self._write_field(field) + self._write_field(field, union) for method in sorted(union.constructors): self._write_constructor(method) for method in sorted(union.methods): @@ -532,7 +553,7 @@ and/or use gtk-doc annotations. ''') for method in sorted(union.static_methods): self._write_static_method(method) - def _write_field(self, field, is_gtype_struct=False): + def _write_field(self, field, parent, is_gtype_struct=False): if field.anonymous_node: if isinstance(field.anonymous_node, ast.Callback): attrs = [('name', field.name)] @@ -544,8 +565,7 @@ and/or use gtk-doc annotations. ''') elif isinstance(field.anonymous_node, ast.Union): self._write_union(field.anonymous_node) else: - raise AssertionError("Unknown field anonymous: %r" \ - % (field.anonymous_node, )) + raise AssertionError("Unknown field anonymous: %r" % (field.anonymous_node, )) else: attrs = [('name', field.name)] self._append_node_generic(field, attrs) @@ -561,7 +581,7 @@ and/or use gtk-doc annotations. ''') attrs.append(('private', '1')) with self.tagcontext('field', attrs): self._write_generic(field) - self._write_type(field.type) + self._write_type(field.type, parent=parent) def _write_signal(self, signal): attrs = [('name', signal.name)] @@ -581,4 +601,4 @@ and/or use gtk-doc annotations. ''') with self.tagcontext('glib:signal', attrs): self._write_generic(signal) self._write_return_type(signal.retval) - self._write_parameters(signal, signal.parameters) + self._write_parameters(signal) diff --git a/giscanner/giscannermodule.c b/giscanner/giscannermodule.c index 0da20f18..b9512275 100644 --- a/giscanner/giscannermodule.c +++ b/giscanner/giscannermodule.c @@ -28,7 +28,6 @@ #ifdef G_OS_WIN32 #define USE_WINDOWS #endif -#include "grealpath.h" #ifdef _WIN32 #include <fcntl.h> @@ -42,8 +41,8 @@ DL_EXPORT(void) init_giscanner(void); -#define NEW_CLASS(ctype, name, cname) \ -static const PyMethodDef _Py##cname##_methods[]; \ +#define NEW_CLASS(ctype, name, cname, num_methods) \ +static const PyMethodDef _Py##cname##_methods[num_methods]; \ PyTypeObject Py##cname##_Type = { \ PyObject_HEAD_INIT(NULL) \ 0, \ @@ -86,9 +85,9 @@ typedef struct { GISourceScanner *scanner; } PyGISourceScanner; -NEW_CLASS (PyGISourceSymbol, "SourceSymbol", GISourceSymbol); -NEW_CLASS (PyGISourceType, "SourceType", GISourceType); -NEW_CLASS (PyGISourceScanner, "SourceScanner", GISourceScanner); +NEW_CLASS (PyGISourceSymbol, "SourceSymbol", GISourceSymbol, 10); +NEW_CLASS (PyGISourceType, "SourceType", GISourceType, 9); +NEW_CLASS (PyGISourceScanner, "SourceScanner", GISourceScanner, 8); /* Symbol */ @@ -162,7 +161,10 @@ symbol_get_const_int (PyGISourceSymbol *self, return Py_None; } - return PyLong_FromLongLong ((long long)self->symbol->const_int); + if (self->symbol->const_int_is_unsigned) + return PyLong_FromUnsignedLongLong ((unsigned long long)self->symbol->const_int); + else + return PyLong_FromLongLong ((long long)self->symbol->const_int); } static PyObject * @@ -191,6 +193,19 @@ symbol_get_const_string (PyGISourceSymbol *self, } static PyObject * +symbol_get_const_boolean (PyGISourceSymbol *self, + void *context) +{ + if (!self->symbol->const_boolean_set) + { + Py_INCREF(Py_None); + return Py_None; + } + + return PyBool_FromLong (self->symbol->const_boolean); +} + +static PyObject * symbol_get_source_filename (PyGISourceSymbol *self, void *context) { @@ -214,6 +229,8 @@ static const PyGetSetDef _PyGISourceSymbol_getsets[] = { /* gboolean const_double_set; */ { "const_double", (getter)symbol_get_const_double, NULL, NULL}, { "const_string", (getter)symbol_get_const_string, NULL, NULL}, + /* gboolean const_boolean_set; */ + { "const_boolean", (getter)symbol_get_const_boolean, NULL, NULL}, { "source_filename", (getter)symbol_get_source_filename, NULL, NULL}, { "line", (getter)symbol_get_line, NULL, NULL}, { "private", (getter)symbol_get_private, NULL, NULL}, @@ -353,12 +370,13 @@ pygi_source_scanner_append_filename (PyGISourceScanner *self, PyObject *args) { char *filename; + GFile *file; if (!PyArg_ParseTuple (args, "s:SourceScanner.append_filename", &filename)) return NULL; - self->scanner->filenames = g_list_append (self->scanner->filenames, - g_realpath (filename)); + file = g_file_new_for_path (filename); + g_hash_table_add (self->scanner->files, file); Py_INCREF (Py_None); return Py_None; @@ -411,44 +429,80 @@ pygi_source_scanner_parse_file (PyGISourceScanner *self, #ifdef _WIN32 /* The file descriptor passed to us is from the C library Python - * uses. That is msvcr71.dll at least for Python 2.5. This code, at - * least if compiled with mingw, uses msvcrt.dll, so we cannot use - * the file descriptor directly. So perform appropriate magic. + * uses. That is msvcr71.dll for Python 2.5 and msvcr90.dll for + * Python 2.6, 2.7, 3.2 etc; and msvcr100.dll for Python 3.3 and later. + * This code, at least if compiled with mingw, uses + * msvcrt.dll, so we cannot use the file descriptor directly. So + * perform appropriate magic. + */ + + /* If we are using the following combinations, + * we can use the file descriptors directly + * (Not if a build using WDK is used): + * Python 2.6.x/2.7.x with Visual C++ 2008 + * Python 3.1.x/3.2.x with Visual C++ 2008 + * Python 3.3+ with Visual C++ 2010 */ + +#if (defined(_MSC_VER) && !defined(USE_WIN_DDK)) +#if (PY_MAJOR_VERSION==2 && PY_MINOR_VERSION>=6 && (_MSC_VER >= 1500 && _MSC_VER < 1600)) +#define MSVC_USE_FD_DIRECTLY 1 +#elif (PY_MAJOR_VERSION==3 && PY_MINOR_VERSION<=2 && (_MSC_VER >= 1500 && _MSC_VER < 1600)) +#define MSVC_USE_FD_DIRECTLY 1 +#elif (PY_MAJOR_VERSION==3 && PY_MINOR_VERSION>=3 && (_MSC_VER >= 1600 && _MSC_VER < 1700)) +#define MSVC_USE_FD_DIRECTLY 1 +#endif +#endif + +#ifndef MSVC_USE_FD_DIRECTLY { - HMODULE msvcr71; - int (*p__get_osfhandle) (int); +#if defined(PY_MAJOR_VERSION) && PY_MAJOR_VERSION==2 && PY_MINOR_VERSION==5 +#define PYTHON_MSVCRXX_DLL "msvcr71.dll" +#elif defined(PY_MAJOR_VERSION) && PY_MAJOR_VERSION==2 && PY_MINOR_VERSION==6 +#define PYTHON_MSVCRXX_DLL "msvcr90.dll" +#elif defined(PY_MAJOR_VERSION) && PY_MAJOR_VERSION==2 && PY_MINOR_VERSION==7 +#define PYTHON_MSVCRXX_DLL "msvcr90.dll" +#elif defined(PY_MAJOR_VERSION) && PY_MAJOR_VERSION==3 && PY_MINOR_VERSION==2 +#define PYTHON_MSVCRXX_DLL "msvcr90.dll" +#elif defined(PY_MAJOR_VERSION) && PY_MAJOR_VERSION==3 && PY_MINOR_VERSION>=3 +#define PYTHON_MSVCRXX_DLL "msvcr100.dll" +#else +#error This Python version not handled +#endif + HMODULE msvcrxx; + intptr_t (*p__get_osfhandle) (int); HANDLE handle; - msvcr71 = GetModuleHandle ("msvcr71.dll"); - if (!msvcr71) - { - g_print ("No msvcr71.dll loaded.\n"); - return NULL; - } + msvcrxx = GetModuleHandle (PYTHON_MSVCRXX_DLL); + if (!msvcrxx) + { + g_print ("No " PYTHON_MSVCRXX_DLL " loaded.\n"); + return NULL; + } - p__get_osfhandle = GetProcAddress (msvcr71, "_get_osfhandle"); + p__get_osfhandle = (intptr_t (*) (int)) GetProcAddress (msvcrxx, "_get_osfhandle"); if (!p__get_osfhandle) - { - g_print ("No _get_osfhandle found in msvcr71.dll.\n"); - return NULL; - } + { + g_print ("No _get_osfhandle found in " PYTHON_MSVCRXX_DLL ".\n"); + return NULL; + } - handle = p__get_osfhandle (fd); + handle = (HANDLE) p__get_osfhandle (fd); if (!p__get_osfhandle) - { - g_print ("Could not get OS handle from msvcr71 fd.\n"); - return NULL; - } + { + g_print ("Could not get OS handle from " PYTHON_MSVCRXX_DLL " fd.\n"); + return NULL; + } - fd = _open_osfhandle (handle, _O_RDONLY); + fd = _open_osfhandle ((intptr_t) handle, _O_RDONLY); if (fd == -1) - { - g_print ("Could not open C fd from OS handle.\n"); - return NULL; - } + { + g_print ("Could not open C fd from OS handle.\n"); + return NULL; + } } #endif +#endif fp = fdopen (fd, "r"); if (!fp) @@ -472,18 +526,19 @@ pygi_source_scanner_lex_filename (PyGISourceScanner *self, PyObject *args) { char *filename; + GFile *file; if (!PyArg_ParseTuple (args, "s:SourceScanner.lex_filename", &filename)) return NULL; - self->scanner->current_filename = g_strdup (filename); + self->scanner->current_file = g_file_new_for_path (filename); if (!gi_source_scanner_lex_filename (self->scanner, filename)) { g_print ("Something went wrong during lexing.\n"); return NULL; } - self->scanner->filenames = - g_list_append (self->scanner->filenames, g_strdup (filename)); + file = g_file_new_for_path (filename); + g_hash_table_add (self->scanner->files, file); Py_INCREF (Py_None); return Py_None; @@ -520,6 +575,7 @@ pygi_source_scanner_get_symbols (PyGISourceScanner *self) PyList_SetItem (list, i++, item); } + g_slist_free (symbols); Py_INCREF (list); return list; } @@ -543,6 +599,7 @@ pygi_source_scanner_get_comments (PyGISourceScanner *self) PyList_SetItem (list, i++, item); } + g_slist_free (comments); Py_INCREF (list); return list; } @@ -656,7 +713,7 @@ pygi_collect_attributes (PyObject *self, goto out; } - if (!PyTuple_Size (tuple) == 2) + if (PyTuple_Size (tuple) != 2) { PyErr_SetString(PyExc_IndexError, "attribute item must be a tuple of length 2"); @@ -722,8 +779,6 @@ init_giscanner(void) PyObject *m, *d; gboolean is_uninstalled; - g_type_init (); - /* Hack to avoid having to create a fake directory structure; when * running uninstalled, the module will be in the top builddir, * with no _giscanner prefix. diff --git a/giscanner/grealpath.h b/giscanner/grealpath.h deleted file mode 100644 index 176f57ec..00000000 --- a/giscanner/grealpath.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef __G_REALPATH_H__ -#define __G_REALPATH_H__ - -#include <stdlib.h> -#ifdef USE_WINDOWS -#include <windows.h> -#endif - -/** - * g_realpath: - * - * this should be a) filled in for win32 and b) put in glib... - */ - -static inline gchar* -g_realpath (const char *path) -{ -#ifndef _WIN32 -#ifndef PATH_MAX -#define PATH_MAX 4096 -#endif - char buffer [PATH_MAX]; - if (realpath(path, buffer)) - return g_strdup(buffer); - else - return NULL; -#else - /* We don't want to include <windows.h> as it clashes horribly - * with token names from scannerparser.h. So just declare - * GetFullPathNameA() here unless we already defined it, like - * in giscanner.c. - */ -#ifndef USE_WINDOWS - extern __stdcall GetFullPathNameA(const char*, int, char*, char**); -#endif - char *buffer; - char dummy; - int rc, len; - - rc = GetFullPathNameA(path, 1, &dummy, NULL); - - if (rc == 0) - { - /* Weird failure, so just return the input path as such */ - return g_strdup(path); - } - - len = rc + 1; - buffer = g_malloc(len); - - rc = GetFullPathNameA(path, len, buffer, NULL); - - if (rc == 0 || rc > len) - { - /* Weird failure again */ - g_free(buffer); - return g_strdup(path); - } - - return buffer; -#endif -} - -#endif diff --git a/giscanner/introspectablepass.py b/giscanner/introspectablepass.py index 97ccfe71..3d67c73e 100644 --- a/giscanner/introspectablepass.py +++ b/giscanner/introspectablepass.py @@ -21,6 +21,7 @@ from . import ast from . import message from .annotationparser import TAG_RETURNS + class IntrospectablePass(object): def __init__(self, transformer, blocks): @@ -58,7 +59,7 @@ class IntrospectablePass(object): else: context = "return value: " if block: - return_tag = block.get_tag(TAG_RETURNS) + return_tag = block.tags.get(TAG_RETURNS) if return_tag: position = return_tag.position message.warn_node(parent, prefix + context + text, @@ -79,7 +80,7 @@ class IntrospectablePass(object): if not node.type.resolved: self._parameter_warning(parent, node, -"Unresolved type: %r" % (node.type.unresolved_string, )) + "Unresolved type: %r" % (node.type.unresolved_string, )) parent.introspectable = False return @@ -87,23 +88,24 @@ class IntrospectablePass(object): parent.introspectable = False return - if (isinstance(node.type, ast.List) - and node.type.element_type == ast.TYPE_ANY): + if (isinstance(node.type, (ast.List, ast.Array)) + and node.type.element_type == ast.TYPE_ANY): self._parameter_warning(parent, node, "Missing (element-type) annotation") parent.introspectable = False return if (is_parameter - and isinstance(target, ast.Callback) - and not node.type.target_giname in ('GLib.DestroyNotify', - 'Gio.AsyncReadyCallback') - and node.scope is None): - self._parameter_warning(parent, node, - ("Missing (scope) annotation for callback" + - " without GDestroyNotify (valid: %s, %s)") - % (ast.PARAM_SCOPE_CALL, ast.PARAM_SCOPE_ASYNC)) - parent.introspectable = False - return + and isinstance(target, ast.Callback) + and not node.type.target_giname in ('GLib.DestroyNotify', 'Gio.AsyncReadyCallback') + and node.scope is None): + self._parameter_warning( + parent, + node, + "Missing (scope) annotation for callback without " + "GDestroyNotify (valid: %s, %s)" % (ast.PARAM_SCOPE_CALL, ast.PARAM_SCOPE_ASYNC)) + + parent.introspectable = False + return if is_return and isinstance(target, ast.Callback): self._parameter_warning(parent, node, "Callbacks cannot be return values; use (skip)") @@ -111,12 +113,14 @@ class IntrospectablePass(object): return if (is_return - and isinstance(target, (ast.Record, ast.Union)) - and target.get_type is None - and not target.foreign): + and isinstance(target, (ast.Record, ast.Union)) + and target.get_type is None + and not target.foreign): if node.transfer != ast.PARAM_TRANSFER_NONE: - self._parameter_warning(parent, node, -"Invalid non-constant return of bare structure or union; register as boxed type or (skip)") + self._parameter_warning( + parent, node, + "Invalid non-constant return of bare structure or union; " + "register as boxed type or (skip)") parent.introspectable = False return @@ -143,10 +147,10 @@ class IntrospectablePass(object): # These are not introspectable pending us adding # larger type tags to the typelib (in theory these could # be 128 bit or larger) - if typeval.is_equiv((ast.TYPE_LONG_LONG, ast.TYPE_LONG_ULONG, - ast.TYPE_LONG_DOUBLE)): + elif typeval.is_equiv((ast.TYPE_LONG_LONG, ast.TYPE_LONG_ULONG, ast.TYPE_LONG_DOUBLE)): return False - return True + else: + return True target = self._transformer.lookup_typenode(typeval) if not target: return False @@ -229,8 +233,8 @@ class IntrospectablePass(object): def _remove_non_reachable_backcompat_copies(self, obj, stack): if obj.skip: return False - if (isinstance(obj, ast.Function) - and not obj.introspectable - and obj.moved_to is not None): - self._namespace.remove(obj) + if (isinstance(obj, ast.Function) and obj.moved_to is not None): + # remove functions that are not introspectable + if not obj.introspectable: + obj.internal_skipped = True return True diff --git a/giscanner/libtoolimporter.py b/giscanner/libtoolimporter.py index 20bd0053..0d26b0c5 100644 --- a/giscanner/libtoolimporter.py +++ b/giscanner/libtoolimporter.py @@ -72,5 +72,5 @@ class LibtoolImporter(object): sys.meta_path.append(cls) @classmethod - def __exit__(cls, type, value, traceback): + def __exit__(cls, exc_type, exc_val, exc_tb): sys.meta_path.remove(cls) diff --git a/giscanner/maintransformer.py b/giscanner/maintransformer.py index d4163fae..c107beed 100644 --- a/giscanner/maintransformer.py +++ b/giscanner/maintransformer.py @@ -21,23 +21,19 @@ import re from . import ast from . import message -from .annotationparser import (TAG_VFUNC, TAG_SINCE, TAG_DEPRECATED, TAG_RETURNS, - TAG_ATTRIBUTES, TAG_RENAME_TO, TAG_TYPE, - TAG_UNREF_FUNC, TAG_REF_FUNC, TAG_SET_VALUE_FUNC, - TAG_GET_VALUE_FUNC, TAG_VALUE, TAG_TRANSFER, - TAG_STABILITY) -from .annotationparser import (OPT_ALLOW_NONE, OPT_ARRAY, OPT_ATTRIBUTE, - OPT_ELEMENT_TYPE, OPT_IN, OPT_INOUT, - OPT_INOUT_ALT, OPT_OUT, OPT_SCOPE, - OPT_OUT_CALLER_ALLOCATES, OPT_OUT_CALLEE_ALLOCATES, - OPT_TYPE, OPT_CLOSURE, OPT_DESTROY, OPT_TRANSFER, OPT_SKIP, - OPT_FOREIGN, OPT_ARRAY_FIXED_SIZE, - OPT_ARRAY_LENGTH, OPT_ARRAY_ZERO_TERMINATED, - OPT_CONSTRUCTOR, OPT_METHOD, - OPT_TRANSFER_NONE, OPT_TRANSFER_FLOATING) -from .annotationparser import AnnotationParser -from .transformer import TransformerException -from .utils import to_underscores, to_underscores_noprefix +from .annotationparser import (TAG_DEPRECATED, TAG_SINCE, TAG_STABILITY, TAG_RETURNS) +from .annotationparser import (ANN_ALLOW_NONE, ANN_ARRAY, ANN_ATTRIBUTES, ANN_CLOSURE, + ANN_CONSTRUCTOR, ANN_DESTROY, ANN_ELEMENT_TYPE, ANN_FOREIGN, + ANN_GET_VALUE_FUNC, ANN_IN, ANN_INOUT, ANN_METHOD, ANN_OUT, + ANN_REF_FUNC, ANN_RENAME_TO, ANN_SCOPE, ANN_SET_VALUE_FUNC, + ANN_SKIP, ANN_TRANSFER, ANN_TYPE, ANN_UNREF_FUNC, ANN_VALUE, + ANN_VFUNC) +from .annotationparser import (OPT_ARRAY_FIXED_SIZE, OPT_ARRAY_LENGTH, OPT_ARRAY_ZERO_TERMINATED, + OPT_OUT_CALLEE_ALLOCATES, OPT_OUT_CALLER_ALLOCATES, + OPT_TRANSFER_FLOATING, OPT_TRANSFER_NONE) + +from .utils import to_underscores_noprefix + class MainTransformer(object): @@ -50,12 +46,10 @@ class MainTransformer(object): # Public API def transform(self): - contents = list(self._namespace.itervalues()) - if len(contents) == 0: - message.fatal("""Namespace is empty; likely causes are: -* Not including .h files to be scanned -* Broken --identifier-prefix -""") + if not self._namespace.names: + message.fatal('Namespace is empty; likely causes are:\n' + '* Not including .h files to be scanned\n' + '* Broken --identifier-prefix') # Some initial namespace surgery self._namespace.walk(self._pass_fixup_hidden_fields) @@ -109,23 +103,21 @@ class MainTransformer(object): def _pass_fixup_hidden_fields(self, node, chain): """Hide all callbacks starting with _; the typical -usage is void (*_gtk_reserved1)(void);""" - if not isinstance(node, (ast.Class, ast.Interface, - ast.Record, ast.Union)): - return True - for field in node.fields: - if field is None: - continue - if (field.name.startswith('_') + usage is void (*_gtk_reserved1)(void);""" + if isinstance(node, (ast.Class, ast.Interface, ast.Record, ast.Union)): + for field in node.fields: + if (field + and field.name is not None + and field.name.startswith('_') and field.anonymous_node is not None and isinstance(field.anonymous_node, ast.Callback)): - field.introspectable = False + field.introspectable = False return True def _get_validate_parameter_name(self, parent, param_name, origin): try: param = parent.get_parameter(param_name) - except ValueError, e: + except ValueError: param = None if param is None: if isinstance(origin, ast.Parameter): @@ -139,30 +131,43 @@ usage is void (*_gtk_reserved1)(void);""" return param.argname + def _get_validate_field_name(self, parent, field_name, origin): + try: + field = parent.get_field(field_name) + except ValueError: + field = None + if field is None: + origin_name = 'field %s' % (origin.name, ) + message.log_node( + message.FATAL, parent, + "can't find field %s referenced by %s of %r" + % (field_name, origin_name, parent.name)) + + return field.name + def _apply_annotation_rename_to(self, node, chain, block): if not block: return - rename_to = block.get_tag(TAG_RENAME_TO) + rename_to = block.annotations.get(ANN_RENAME_TO) if not rename_to: return - rename_to = rename_to.value + rename_to = rename_to[0] target = self._namespace.get_by_symbol(rename_to) if not target: message.warn_node(node, - "Can't find symbol %r referenced by Rename annotation" % ( - rename_to, )) + "Can't find symbol %r referenced by \"rename-to\" annotation" % (rename_to, )) elif target.shadowed_by: message.warn_node(node, - "Function %r already shadowed by %r, can't overwrite with %r" % ( - target.symbol, - target.shadowed_by, - rename_to)) + "Function %r already shadowed by %r, can't overwrite " + "with %r" % (target.symbol, + target.shadowed_by, + rename_to)) elif target.shadows: message.warn_node(node, - "Function %r already shadows %r, can't multiply shadow with %r" % ( - target.symbol, - target.shadows, - rename_to)) + "Function %r already shadows %r, can't multiply shadow " + "with %r" % (target.symbol, + target.shadows, + rename_to)) else: target.shadowed_by = node.name node.shadows = target.name @@ -192,7 +197,7 @@ usage is void (*_gtk_reserved1)(void);""" def _get_annotation_name(self, node): if isinstance(node, (ast.Class, ast.Interface, ast.Record, ast.Union, ast.Enum, ast.Bitfield, - ast.Callback, ast.Alias)): + ast.Callback, ast.Alias, ast.Constant)): if node.ctype is not None: return node.ctype elif isinstance(node, ast.Registered) and node.gtype_name is not None: @@ -211,19 +216,21 @@ usage is void (*_gtk_reserved1)(void);""" if isinstance(node, ast.Function): self._apply_annotations_function(node, chain) if isinstance(node, ast.Callback): - self._apply_annotations_callable(node, chain, block = self._get_block(node)) + self._apply_annotations_callable(node, chain, block=self._get_block(node)) if isinstance(node, (ast.Class, ast.Interface, ast.Union, ast.Enum, ast.Bitfield, ast.Callback)): self._apply_annotations_annotated(node, self._get_block(node)) + if isinstance(node, (ast.Enum, ast.Bitfield)): + self._apply_annotations_enum_members(node, self._get_block(node)) if isinstance(node, (ast.Class, ast.Interface, ast.Record, ast.Union)): block = self._get_block(node) for field in node.fields: self._apply_annotations_field(node, block, field) name = self._get_annotation_name(node) - section_name = 'SECTION:' + name.lower() + section_name = 'SECTION:%s' % (name.lower(), ) block = self._blocks.get(section_name) - if block: - node.doc = block.comment + if block and block.description: + node.doc = block.description if isinstance(node, (ast.Class, ast.Interface)): for prop in node.properties: self._apply_annotations_property(node, prop) @@ -232,29 +239,26 @@ usage is void (*_gtk_reserved1)(void);""" if isinstance(node, ast.Class): block = self._get_block(node) if block: - tag = block.get_tag(TAG_UNREF_FUNC) - node.unref_func = tag.value if tag else None - tag = block.get_tag(TAG_REF_FUNC) - node.ref_func = tag.value if tag else None - tag = block.get_tag(TAG_SET_VALUE_FUNC) - node.set_value_func = tag.value if tag else None - tag = block.get_tag(TAG_GET_VALUE_FUNC) - node.get_value_func = tag.value if tag else None + annotation = block.annotations.get(ANN_UNREF_FUNC) + node.unref_func = annotation[0] if annotation else None + annotation = block.annotations.get(ANN_REF_FUNC) + node.ref_func = annotation[0] if annotation else None + annotation = block.annotations.get(ANN_SET_VALUE_FUNC) + node.set_value_func = annotation[0] if annotation else None + annotation = block.annotations.get(ANN_GET_VALUE_FUNC) + node.get_value_func = annotation[0] if annotation else None if isinstance(node, ast.Constant): self._apply_annotations_constant(node) return True - def _adjust_container_type(self, parent, node, options): - has_element_type = OPT_ELEMENT_TYPE in options - has_array = OPT_ARRAY in options - - if has_array: - self._apply_annotations_array(parent, node, options) - elif has_element_type: - self._apply_annotations_element_type(parent, node, options) + def _adjust_container_type(self, parent, node, annotations): + if ANN_ARRAY in annotations: + self._apply_annotations_array(parent, node, annotations) + elif ANN_ELEMENT_TYPE in annotations: + self._apply_annotations_element_type(parent, node, annotations) if isinstance(node.type, ast.Array): - self._check_array_element_type(node.type, options) + self._check_array_element_type(node.type, annotations) def _resolve(self, type_str, type_node=None, node=None, parent=None): def grab_one(type_str, resolver, top_combiner, combiner): @@ -262,7 +266,7 @@ usage is void (*_gtk_reserved1)(void);""" Use resolver() on each identifier, and combiner() on the parts of each complete type. (top_combiner is used on the top-most type.)""" bits = re.split(r'([,<>()])', type_str, 1) - first, sep, rest = [bits[0], '', ''] if (len(bits)==1) else bits + first, sep, rest = [bits[0], '', ''] if (len(bits) == 1) else bits args = [resolver(first)] if sep == '<' or sep == '(': lastsep = '>' if (sep == '<') else ')' @@ -273,9 +277,11 @@ usage is void (*_gtk_reserved1)(void);""" else: rest = sep + rest return top_combiner(*args), rest + def resolver(ident): res = self._transformer.create_type_from_user_string(ident) return res + def combiner(base, *rest): if not rest: return base @@ -286,6 +292,7 @@ usage is void (*_gtk_reserved1)(void);""" message.warn( "Too many parameters in type specification %r" % (type_str, )) return base + def top_combiner(base, *rest): if type_node is not None and isinstance(type_node, ast.Type): base.is_const = type_node.is_const @@ -304,7 +311,7 @@ usage is void (*_gtk_reserved1)(void);""" else: text = type_str message.warn_node(parent, "%s: Unknown type: %r" % - (text, result.ctype), positions=position) + (text, type_str), positions=position) return result def _resolve_toplevel(self, type_str, type_node=None, node=None, parent=None): @@ -320,50 +327,43 @@ usage is void (*_gtk_reserved1)(void);""" block = self._blocks.get(func.symbol) if block: if isinstance(param, ast.Parameter): - tag = block.params.get(param.argname) + part = block.params.get(param.argname) elif isinstance(param, ast.Return): - tag = block.tags.get(TAG_RETURNS) + part = block.tags.get(TAG_RETURNS) else: - tag = None + part = None - if tag.position: - return tag.position + if part.position: + return part.position return block.position - def _check_array_element_type(self, array, options): + def _check_array_element_type(self, array, annotations): + array_type = array.array_type + element_type = array.element_type + # GPtrArrays are allowed to contain non basic types # (except enums and flags) or basic types that are # as big as a gpointer - if array.array_type == ast.Array.GLIB_PTRARRAY and \ - ((array.element_type in ast.BASIC_GIR_TYPES - and not array.element_type in ast.POINTER_TYPES) or - isinstance(array.element_type, ast.Enum) or - isinstance(array.element_type, ast.Bitfield)): - message.warn("invalid (element-type) for a GPtrArray, " - "must be a pointer", options.position) + if array_type == ast.Array.GLIB_PTRARRAY: + if ((element_type in ast.BASIC_GIR_TYPES and not element_type in ast.POINTER_TYPES) + or isinstance(element_type, (ast.Enum, ast.Bitfield))): + message.warn("invalid (element-type) for a GPtrArray, " + "must be a pointer", annotations.position) # GByteArrays have (element-type) guint8 by default - if array.array_type == ast.Array.GLIB_BYTEARRAY: - if array.element_type == ast.TYPE_ANY: + if array_type == ast.Array.GLIB_BYTEARRAY: + if element_type == ast.TYPE_ANY: array.element_type = ast.TYPE_UINT8 - elif not array.element_type in [ast.TYPE_UINT8, - ast.TYPE_INT8, - ast.TYPE_CHAR]: + elif not element_type in [ast.TYPE_UINT8, ast.TYPE_INT8, ast.TYPE_CHAR]: message.warn("invalid (element-type) for a GByteArray, " "must be one of guint8, gint8 or gchar", - options.position) - - def _apply_annotations_array(self, parent, node, options): - array_opt = options.get(OPT_ARRAY) - if array_opt: - array_values = array_opt.all() - else: - array_values = {} + annotations.position) - element_type = options.get(OPT_ELEMENT_TYPE) - if element_type is not None: - element_type_node = self._resolve(element_type.one(), + def _apply_annotations_array(self, parent, node, annotations): + element_type_options = annotations.get(ANN_ELEMENT_TYPE) + if element_type_options: + element_type_node = self._resolve(element_type_options[0], node.type, node, parent) elif isinstance(node.type, ast.Array): element_type_node = node.type.element_type @@ -372,82 +372,82 @@ usage is void (*_gtk_reserved1)(void);""" # and no (element-type) means array of Foo element_type_node = node.type.clone() # The element's ctype is the array's dereferenced - if element_type_node.ctype is not None and \ - element_type_node.ctype.endswith('*'): + if element_type_node.ctype is not None and element_type_node.ctype.endswith('*'): element_type_node.ctype = element_type_node.ctype[:-1] if isinstance(node.type, ast.Array): array_type = node.type.array_type else: array_type = None - container_type = ast.Array(array_type, element_type_node, - ctype=node.type.ctype, - is_const=node.type.is_const) - if OPT_ARRAY_ZERO_TERMINATED in array_values: - container_type.zeroterminated = array_values.get( - OPT_ARRAY_ZERO_TERMINATED) == '1' + + array_options = annotations.get(ANN_ARRAY) + container_type = ast.Array(array_type, element_type_node, ctype=node.type.ctype, + is_const=node.type.is_const) + if OPT_ARRAY_ZERO_TERMINATED in array_options: + container_type.zeroterminated = array_options.get(OPT_ARRAY_ZERO_TERMINATED) == '1' else: container_type.zeroterminated = False - length = array_values.get(OPT_ARRAY_LENGTH) - if length is not None: - paramname = self._get_validate_parameter_name(parent, length, node) + + length = array_options.get(OPT_ARRAY_LENGTH) + if length: + if isinstance(parent, ast.Compound): + paramname = self._get_validate_field_name(parent, length, node) + else: + paramname = self._get_validate_parameter_name(parent, length, node) + if paramname: + param = parent.get_parameter(paramname) + param.direction = node.direction + if param.direction == ast.PARAM_DIRECTION_OUT: + param.transfer = ast.PARAM_TRANSFER_FULL if paramname: - param = parent.get_parameter(paramname) - param.direction = node.direction - if param.direction == ast.PARAM_DIRECTION_OUT: - param.transfer = ast.PARAM_TRANSFER_FULL - container_type.length_param_name = param.argname - fixed = array_values.get(OPT_ARRAY_FIXED_SIZE) + container_type.length_param_name = paramname + fixed = array_options.get(OPT_ARRAY_FIXED_SIZE) if fixed: try: container_type.size = int(fixed) - except ValueError: + except (TypeError, ValueError): # Already warned in annotationparser.py return node.type = container_type - def _apply_annotations_element_type(self, parent, node, options): - element_type_opt = options.get(OPT_ELEMENT_TYPE) - if element_type_opt is None: - message.warn( - 'element-type annotation takes at least one option, ' - 'none given', - options.position) + def _apply_annotations_element_type(self, parent, node, annotations): + element_type_options = annotations.get(ANN_ELEMENT_TYPE) + if element_type_options is None: return if isinstance(node.type, ast.List): - if element_type_opt.length() != 1: + if len(element_type_options) != 1: message.warn( - 'element-type annotation for a list must have exactly ' - 'one option, not %d options' % (element_type_opt.length(), ), - options.position) + '"element-type" annotation for a list must have exactly ' + 'one option, not %d options' % (len(element_type_options), ), + annotations.position) return - node.type.element_type = self._resolve(element_type_opt.one(), + node.type.element_type = self._resolve(element_type_options[0], node.type, node, parent) elif isinstance(node.type, ast.Map): - if element_type_opt.length() != 2: + if len(element_type_options) != 2: message.warn( - 'element-type annotation for a hash table must have exactly ' - 'two options, not %d option(s)' % (element_type_opt.length(), ), - options.position) + '"element-type" annotation for a hash table must have exactly ' + 'two options, not %d option(s)' % (len(element_type_options), ), + annotations.position) return - element_type = element_type_opt.flat() - node.type.key_type = self._resolve(element_type[0], + node.type.key_type = self._resolve(element_type_options[0], node.type, node, parent) - node.type.value_type = self._resolve(element_type[1], + node.type.value_type = self._resolve(element_type_options[1], node.type, node, parent) elif isinstance(node.type, ast.Array): - if element_type_opt.length() != 1: + if len(element_type_options) != 1: message.warn( - 'element-type annotation for an array must have exactly ' - 'one option, not %d options' % (element_type_opt.length(), ), - options.position) + '"element-type" annotation for an array must have exactly ' + 'one option, not %d options' % (len(element_type_options), ), + annotations.position) return - node.type.element_type = self._resolve(element_type_opt.one(), + node.type.element_type = self._resolve(element_type_options[0], node.type, node, parent) else: - message.warn_node(parent, - "Unknown container %r for element-type annotation" % (node.type, )) + message.warn( + "Unknown container %r for element-type annotation" % (node.type, ), + annotations.position) def _get_transfer_default_param(self, parent, node): if node.direction in [ast.PARAM_DIRECTION_INOUT, @@ -459,8 +459,8 @@ usage is void (*_gtk_reserved1)(void);""" def _get_transfer_default_returntype_basic(self, typeval): if (typeval.is_equiv(ast.BASIC_GIR_TYPES) - or typeval.is_const - or typeval.is_equiv(ast.TYPE_NONE)): + or typeval.is_const + or typeval.is_equiv(ast.TYPE_NONE)): return ast.PARAM_TRANSFER_NONE elif typeval.is_equiv(ast.TYPE_STRING): # Non-const strings default to FULL @@ -477,8 +477,8 @@ usage is void (*_gtk_reserved1)(void);""" assert supercls if cls is supercls: return True - if cls.parent and cls.parent.target_giname != 'GObject.Object': - return self._is_gi_subclass(cls.parent, supercls_type) + if cls.parent_type and cls.parent_type.target_giname != 'GObject.Object': + return self._is_gi_subclass(cls.parent_type, supercls_type) return False def _get_transfer_default_return(self, parent, node): @@ -531,24 +531,22 @@ usage is void (*_gtk_reserved1)(void);""" raise AssertionError(node) def _apply_annotations_param_ret_common(self, parent, node, tag): - options = getattr(tag, 'options', {}) + annotations = tag.annotations if tag else {} - param_type = options.get(OPT_TYPE) - if param_type: - node.type = self._resolve_toplevel(param_type.one(), + type_annotation = annotations.get(ANN_TYPE) + if type_annotation: + node.type = self._resolve_toplevel(type_annotation[0], node.type, node, parent) caller_allocates = False annotated_direction = None - if (OPT_INOUT in options or - OPT_INOUT_ALT in options): + if ANN_INOUT in annotations: annotated_direction = ast.PARAM_DIRECTION_INOUT - elif OPT_OUT in options: - subtype = options[OPT_OUT] - if subtype is not None: - subtype = subtype.one() + elif ANN_OUT in annotations: annotated_direction = ast.PARAM_DIRECTION_OUT - if subtype in (None, ''): + + options = annotations[ANN_OUT] + if len(options) == 0: if node.type.target_giname and node.type.ctype: target = self._transformer.lookup_giname(node.type.target_giname) target = self._transformer.resolve_aliases(target) @@ -557,11 +555,13 @@ usage is void (*_gtk_reserved1)(void);""" caller_allocates = (not has_double_indirection and is_structure_or_union) else: caller_allocates = False - elif subtype == OPT_OUT_CALLER_ALLOCATES: - caller_allocates = True - elif subtype == OPT_OUT_CALLEE_ALLOCATES: - caller_allocates = False - elif OPT_IN in options: + else: + option = options[0] + if option == OPT_OUT_CALLER_ALLOCATES: + caller_allocates = True + elif option == OPT_OUT_CALLEE_ALLOCATES: + caller_allocates = False + elif ANN_IN in annotations: annotated_direction = ast.PARAM_DIRECTION_IN if (annotated_direction is not None) and (annotated_direction != node.direction): @@ -570,79 +570,77 @@ usage is void (*_gtk_reserved1)(void);""" # Also reset the transfer default if we're toggling direction node.transfer = self._get_transfer_default(parent, node) - transfer_tag = options.get(OPT_TRANSFER) - if transfer_tag and transfer_tag.length() == 1: - transfer = transfer_tag.one() + transfer_annotation = annotations.get(ANN_TRANSFER) + if transfer_annotation and len(transfer_annotation) == 1: + transfer = transfer_annotation[0] if transfer == OPT_TRANSFER_FLOATING: transfer = OPT_TRANSFER_NONE node.transfer = transfer - self._adjust_container_type(parent, node, options) + self._adjust_container_type(parent, node, annotations) - if (OPT_ALLOW_NONE in options or - node.type.target_giname == 'Gio.AsyncReadyCallback' or - node.type.target_giname == 'Gio.Cancellable'): + if (ANN_ALLOW_NONE in annotations + or node.type.target_giname == 'Gio.AsyncReadyCallback' + or node.type.target_giname == 'Gio.Cancellable'): node.allow_none = True - if tag is not None and tag.comment is not None: - node.doc = tag.comment + if tag and tag.description: + node.doc = tag.description - if OPT_SKIP in options: + if ANN_SKIP in annotations: node.skip = True - if options: - for attribute in options.getall(OPT_ATTRIBUTE): - node.attributes.append(attribute.flat()) + if annotations: + attributes_annotation = annotations.get(ANN_ATTRIBUTES) + if attributes_annotation is not None: + for key, value in attributes_annotation.items(): + if value: + node.attributes[key] = value def _apply_annotations_annotated(self, node, block): if block is None: return - node.doc = block.comment + if block.description: + node.doc = block.description - since_tag = block.get_tag(TAG_SINCE) + since_tag = block.tags.get(TAG_SINCE) if since_tag is not None: - node.version = since_tag.value + if since_tag.value: + node.version = since_tag.value + if since_tag.description: + node.version_doc = since_tag.description - deprecated_tag = block.get_tag(TAG_DEPRECATED) + deprecated_tag = block.tags.get(TAG_DEPRECATED) if deprecated_tag is not None: - value = deprecated_tag.value - if ': ' in value: - colon = value.find(': ') - version = value[:colon] - desc = value[colon+2:] - else: - desc = value - version = None - node.deprecated = desc - if version is not None: - node.deprecated_version = version + if deprecated_tag.value: + node.deprecated = deprecated_tag.value + if deprecated_tag.description: + node.deprecated_doc = deprecated_tag.description - stability_tag = block.get_tag(TAG_STABILITY) + stability_tag = block.tags.get(TAG_STABILITY) if stability_tag is not None: - stability = stability_tag.value.capitalize() - if stability in ["Stable", "Unstable", "Private", "Internal"]: - node.stability = stability - else: - message.warn('unknown value "%s" for Stability tag' % ( - stability_tag.value), stability_tag.position) - - annos_tag = block.get_tag(TAG_ATTRIBUTES) - if annos_tag is not None: - for key, value in annos_tag.options.iteritems(): + if stability_tag.value: + node.stability = stability_tag.value + if stability_tag.description: + node.stability_doc = stability_tag.description + + attributes_annotation = block.annotations.get(ANN_ATTRIBUTES) + if attributes_annotation is not None: + for key, value in attributes_annotation.items(): if value: - node.attributes.append((key, value.one())) + node.attributes[key] = value - if OPT_SKIP in block.options: + if ANN_SKIP in block.annotations: node.skip = True - if OPT_FOREIGN in block.options: + if ANN_FOREIGN in block.annotations: node.foreign = True - if OPT_CONSTRUCTOR in block.options and isinstance(node, ast.Function): + if ANN_CONSTRUCTOR in block.annotations and isinstance(node, ast.Function): node.is_constructor = True - if OPT_METHOD in block.options: + if ANN_METHOD in block.annotations: node.is_method = True def _apply_annotations_alias(self, node, chain): @@ -650,20 +648,18 @@ usage is void (*_gtk_reserved1)(void);""" self._apply_annotations_annotated(node, block) def _apply_annotations_param(self, parent, param, tag): - if tag: - options = tag.options - else: - options = {} + annotations = tag.annotations if tag else {} + if isinstance(parent, (ast.Function, ast.VFunction)): - scope = options.get(OPT_SCOPE) - if scope and scope.length() == 1: - param.scope = scope.one() + scope_annotation = annotations.get(ANN_SCOPE) + if scope_annotation and len(scope_annotation) == 1: + param.scope = scope_annotation[0] param.transfer = ast.PARAM_TRANSFER_NONE - destroy = options.get(OPT_DESTROY) - if destroy: + destroy_annotation = annotations.get(ANN_DESTROY) + if destroy_annotation: param.destroy_name = self._get_validate_parameter_name(parent, - destroy.one(), + destroy_annotation[0], param) if param.destroy_name is not None: param.scope = ast.PARAM_SCOPE_NOTIFIED @@ -672,13 +668,14 @@ usage is void (*_gtk_reserved1)(void);""" # itself. But this helps avoid tripping a warning from finaltransformer, # since we don't have a way right now to flag this callback a destroy. destroy_param.scope = ast.PARAM_SCOPE_NOTIFIED - closure = options.get(OPT_CLOSURE) - if closure and closure.length() == 1: + + closure_annotation = annotations.get(ANN_CLOSURE) + if closure_annotation and len(closure_annotation) == 1: param.closure_name = self._get_validate_parameter_name(parent, - closure.one(), - param) + closure_annotation[0], + param) elif isinstance(parent, ast.Callback): - if OPT_CLOSURE in options: + if ANN_CLOSURE in annotations: # For callbacks, (closure) appears without an # argument, and tags a parameter that is a closure. We # represent it (weirdly) in the gir and typelib by @@ -689,7 +686,7 @@ usage is void (*_gtk_reserved1)(void);""" def _apply_annotations_return(self, parent, return_, block): if block: - tag = block.get_tag(TAG_RETURNS) + tag = block.tags.get(TAG_RETURNS) else: tag = None self._apply_annotations_param_ret_common(parent, return_, tag) @@ -697,13 +694,19 @@ usage is void (*_gtk_reserved1)(void);""" def _apply_annotations_params(self, parent, params, block): declparams = set([]) if parent.instance_parameter: + if block: + doc_param = block.params.get(parent.instance_parameter.argname) + else: + doc_param = None + self._apply_annotations_param(parent, parent.instance_parameter, doc_param) declparams.add(parent.instance_parameter.argname) + for param in params: if block: - tag = block.get_param(param.argname) + doc_param = block.params.get(param.argname) else: - tag = None - self._apply_annotations_param(parent, param, tag) + doc_param = None + self._apply_annotations_param(parent, param, doc_param) declparams.add(param.argname) if not block: @@ -714,57 +717,38 @@ usage is void (*_gtk_reserved1)(void);""" unused = declparams - docparams for doc_name in unknown: - # Skip varargs, see #629759 - if doc_name.lower() in ['...', 'varargs', TAG_RETURNS]: - continue if len(unused) == 0: text = '' elif len(unused) == 1: (param, ) = unused text = ', should be %r' % (param, ) else: - text = ', should be one of %s' % ( - ', '.join(repr(p) for p in unused), ) + text = ', should be one of %s' % (', '.join(repr(p) for p in unused), ) - tag = block.get_param(doc_name) - message.warn( - '%s: unknown parameter %r in documentation comment%s' % ( - block.name, doc_name, text), - tag.position) + param = block.params.get(doc_name) + message.warn('%s: unknown parameter %r in documentation ' + 'comment%s' % (block.name, doc_name, text), + param.position) def _apply_annotations_callable(self, node, chain, block): self._apply_annotations_annotated(node, block) self._apply_annotations_params(node, node.parameters, block) self._apply_annotations_return(node, node.retval, block) - def _check_arg_annotations(self, parent, params, block): - if block is None: - return - for tag in block.tags.keys(): - if tag == TAG_RETURNS: - continue - for param in params: - if param.argname == tag: - break - else: - message.warn( - "Annotation for '%s' refers to unknown argument '%s'" - % (parent.name, tag)) - def _apply_annotations_field(self, parent, block, field): if not block: return - tag = block.get_param(field.name) + tag = block.params.get(field.name) if not tag: return - t = tag.options.get(OPT_TYPE) - if t: - field.type = self._transformer.create_type_from_user_string(t.one()) - + type_annotation = tag.annotations.get(ANN_TYPE) + if type_annotation: + field.type = self._transformer.create_type_from_user_string(type_annotation[0]) + field.doc = tag.description try: - self._adjust_container_type(parent, field, tag.options) - except AttributeError: - pass + self._adjust_container_type(parent, field, tag.annotations) + except AttributeError, ex: + print ex def _apply_annotations_property(self, parent, prop): prefix = self._get_annotation_name(parent) @@ -772,83 +756,99 @@ usage is void (*_gtk_reserved1)(void);""" self._apply_annotations_annotated(prop, block) if not block: return - transfer_tag = block.get_tag(TAG_TRANSFER) - if transfer_tag is not None: - transfer = transfer_tag.value + transfer_annotation = block.annotations.get(ANN_TRANSFER) + if transfer_annotation is not None: + transfer = transfer_annotation[0] if transfer == OPT_TRANSFER_FLOATING: transfer = OPT_TRANSFER_NONE prop.transfer = transfer else: prop.transfer = self._get_transfer_default(parent, prop) - type_tag = block.get_tag(TAG_TYPE) - if type_tag: - prop.type = self._resolve_toplevel(type_tag.value, prop.type, prop, parent) + type_annotation = block.annotations.get(ANN_TYPE) + if type_annotation: + prop.type = self._resolve_toplevel(type_annotation[0], prop.type, prop, parent) def _apply_annotations_signal(self, parent, signal): + names = [] prefix = self._get_annotation_name(parent) block = self._blocks.get('%s::%s' % (prefix, signal.name)) - self._apply_annotations_annotated(signal, block) - # We're only attempting to name the signal parameters if - # the number of parameter tags (@foo) is the same or greater - # than the number of signal parameters - if block and len(block.params) > len(signal.parameters): - names = block.params.items() - # Resolve real parameter names early, so that in later - # phase we can refer to them while resolving annotations. - for i, param in enumerate(signal.parameters): - param.argname, tag = names[i+1] - else: - names = [] + + if block: + self._apply_annotations_annotated(signal, block) + + # We're only attempting to name the signal parameters if + # the number of parameters (@foo) is the same or greater + # than the number of signal parameters + if len(block.params) > len(signal.parameters): + names = block.params.items() + # Resolve real parameter names early, so that in later + # phase we can refer to them while resolving annotations. + for i, param in enumerate(signal.parameters): + param.argname, tag = names[i + 1] + elif len(signal.parameters) != 0: + # Only warn about missing params if there are actually parameters + # besides implicit self. + message.warn("incorrect number of parameters in comment block, " + "parameter annotations will be ignored.", block.position) + for i, param in enumerate(signal.parameters): if names: - name, tag = names[i+1] - options = getattr(tag, 'options', {}) - param_type = options.get(OPT_TYPE) - if param_type: - param.type = self._resolve_toplevel(param_type.one(), param.type, - param, parent) + name, tag = names[i + 1] + if tag: + type_annotation = tag.annotations.get(ANN_TYPE) + if type_annotation: + param.type = self._resolve_toplevel(type_annotation[0], param.type, + param, parent) else: tag = None self._apply_annotations_param(signal, param, tag) self._apply_annotations_return(signal, signal.retval, block) def _apply_annotations_constant(self, node): - block = self._blocks.get(node.ctype) - if not block: + block = self._get_block(node) + if block is None: return - tag = block.get_tag(TAG_VALUE) - if tag: - node.value = tag.value - def _pass_read_annotations2(self, node, chain): - if isinstance(node, ast.Function): - self._apply_annotations2_function(node, chain) - return True - - def _apply_annotations2_function(self, node, chain): - block = self._blocks.get(node.symbol) + self._apply_annotations_annotated(node, block) - self._apply_annotation_rename_to(node, chain, block) + value_annotation = block.annotations.get(ANN_VALUE) + if value_annotation: + node.value = value_annotation[0] - # Handle virtual invokers - parent = chain[-1] if chain else None - if not (block and parent): - return - virtual = block.get_tag(TAG_VFUNC) - if not virtual: + def _apply_annotations_enum_members(self, node, block): + if block is None: return - invoker_name = virtual.value - matched = False - for vfunc in parent.virtual_methods: - if vfunc.name == invoker_name: - matched = True - vfunc.invoker = node.name - # Also merge in annotations - self._apply_annotations_callable(vfunc, [parent], block) - break - if not matched: - message.warn_node(node, - "Virtual slot %r not found for %r annotation" % (invoker_name, TAG_VFUNC)) + + for m in node.members: + param = block.params.get(m.symbol, None) + if param and param.description: + m.doc = param.description + + def _pass_read_annotations2(self, node, chain): + if isinstance(node, ast.Function): + block = self._blocks.get(node.symbol) + + self._apply_annotation_rename_to(node, chain, block) + + # Handle virtual invokers + parent = chain[-1] if chain else None + if (block and parent): + virtual_annotation = block.annotations.get(ANN_VFUNC) + if virtual_annotation: + invoker_name = virtual_annotation[0] + matched = False + for vfunc in parent.virtual_methods: + if vfunc.name == invoker_name: + matched = True + vfunc.invoker = node.name + # Also merge in annotations + self._apply_annotations_callable(vfunc, [parent], block) + break + if not matched: + message.warn_node(node, + "Virtual slot %r not found for %r annotation" % (invoker_name, + ANN_VFUNC)) + return True def _resolve_and_filter_type_list(self, typelist): """Given a list of Type instances, return a new list of types with @@ -877,19 +877,18 @@ the ones that failed to resolve removed.""" else: self._transformer.resolve_type(field.type) if isinstance(node, (ast.Class, ast.Interface)): - resolved_parent = None for parent in node.parent_chain: try: self._transformer.resolve_type(parent) - except ValueError, e: + except ValueError: continue target = self._transformer.lookup_typenode(parent) if target: - node.parent = parent + node.parent_type = parent break else: if isinstance(node, ast.Interface): - node.parent = ast.Type(target_giname='GObject.Object') + node.parent_type = ast.Type(target_giname='GObject.Object') for prop in node.properties: self._transformer.resolve_type(prop.type) for sig in node.signals: @@ -1156,9 +1155,9 @@ method or constructor of some type.""" origin_node = self._get_constructor_class(func, subsymbol) if origin_node is None: if func.is_constructor: - message.warn_node(func, - "Can't find matching type for constructor; symbol=%r" \ - % (func.symbol, )) + message.warn_node( + func, + "Can't find matching type for constructor; symbol=%r" % (func.symbol, )) return False # Some sanity checks; only objects and boxeds can have ctors @@ -1184,26 +1183,26 @@ method or constructor of some type.""" while parent and (not parent.gi_name == 'GObject.Object'): if parent == target: break - if parent.parent: - parent = self._transformer.lookup_typenode(parent.parent) + if parent.parent_type: + parent = self._transformer.lookup_typenode(parent.parent_type) else: parent = None if parent is None: message.warn_node(func, - "Return value is not superclass for constructor; " - "symbol=%r constructed=%r return=%r" % ( - func.symbol, - str(origin_node.create_type()), - str(func.retval.type))) + "Return value is not superclass for constructor; " + "symbol=%r constructed=%r return=%r" % + (func.symbol, + str(origin_node.create_type()), + str(func.retval.type))) return False else: if origin_node != target: message.warn_node(func, - "Constructor return type mismatch symbol=%r " - "constructed=%r return=%r" % ( - func.symbol, - str(origin_node.create_type()), - str(func.retval.type))) + "Constructor return type mismatch symbol=%r " + "constructed=%r return=%r" % + (func.symbol, + str(origin_node.create_type()), + str(func.retval.type))) return False return True @@ -1282,7 +1281,7 @@ method or constructor of some type.""" params = node.parameters # First, do defaults for well-known callback types - for i, param in enumerate(params): + for param in params: argnode = self._transformer.lookup_typenode(param.type) if isinstance(argnode, ast.Callback): if param.type.target_giname in ('Gio.AsyncReadyCallback', @@ -1291,7 +1290,7 @@ method or constructor of some type.""" param.transfer = ast.PARAM_TRANSFER_NONE callback_param = None - for i, param in enumerate(params): + for param in params: argnode = self._transformer.lookup_typenode(param.type) is_destroynotify = False if isinstance(argnode, ast.Callback): diff --git a/giscanner/mallard-C-class.tmpl b/giscanner/mallard-C-class.tmpl deleted file mode 100644 index 2d739043..00000000 --- a/giscanner/mallard-C-class.tmpl +++ /dev/null @@ -1,48 +0,0 @@ -<?xml version="1.0"?> -<page id="${node.namespace.name}.${node.name}" - type="guide" - style="class" - xmlns="http://projectmallard.org/1.0/" - xmlns:api="http://projectmallard.org/experimental/api/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="index" group="class"/> - </info> - <title>${node.ctype}</title> -${formatter.format(node.doc)} -% if node.version: -<p>Since ${node.version}</p> -% endif - <synopsis ui:expanded="no"> - <title>Hierarchy</title> - <tree> - <item> - <code>GObjectObject</code> - </item> - </tree> - </synopsis> - <links type="topic" ui:expanded="yes" - api:type="function" api:mime="text/x-csrc" - groups="constructor" style="linklist"> - <title>Constructors</title> - </links> - <links type="topic" ui:expanded="yes" - api:type="function" api:mime="text/x-csrc" - groups="method" style="linklist"> - <title>Methods</title> - </links> - <links type="topic" ui:expanded="yes" - api:type="function" api:mime="text/x-csrc" - groups="function" style="linklist"> - <title>Functions</title> - </links> - <links type="topic" ui:expanded="yes" groups="property" style="linklist"> - <title>Properties</title> - </links> - <links type="topic" ui:expanded="yes" groups="signal" style="linklist"> - <title>Signals</title> - </links> - <links type="topic" ui:expanded="yes" groups="#first #default #last" style="linklist"> - <title>Other</title> - </links> -</page> diff --git a/giscanner/mallard-C-default.tmpl b/giscanner/mallard-C-default.tmpl deleted file mode 100644 index 577fa566..00000000 --- a/giscanner/mallard-C-default.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -<?xml version="1.0"?> -<page id="${namespace.name}.${node.name}" - type="topic" - style="" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - </info> - <title>${namespace.name}.${node.name}</title> -${formatter.format(node.doc)} -</page> diff --git a/giscanner/mallard-C-enum.tmpl b/giscanner/mallard-C-enum.tmpl deleted file mode 100644 index 20cd0894..00000000 --- a/giscanner/mallard-C-enum.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -<?xml version="1.0"?> -<page id="${node.namespace.name}.${node.name}" - type="guide" - style="enum" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="index"/> - </info> - <title>${node.namespace.name}.${node.name}</title> -${formatter.format(node.doc)} -</page> diff --git a/giscanner/mallard-C-function.tmpl b/giscanner/mallard-C-function.tmpl deleted file mode 100644 index 2da4710f..00000000 --- a/giscanner/mallard-C-function.tmpl +++ /dev/null @@ -1,95 +0,0 @@ -<?xml version="1.0"?> -<% -page_style = 'function' -if node.is_constructor: - page_style = 'constructor' -elif node.is_method: - page_style = 'method' -%> -<page id="${page_id}" - type="topic" - style="${page_style}" - xmlns="http://projectmallard.org/1.0/" - xmlns:api="http://projectmallard.org/experimental/api/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> -% if node.parent is not None: - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="${page_style}"/> -% else: - <link type="guide" xref="index" group="${page_style}"/> -% endif - <api:function> - <api:returns> - <api:type>${formatter.format_type(node.retval.type) | x}</api:type> - </api:returns> - <api:name>${node.symbol}</api:name> -% if node.is_method: - <api:arg> - <api:type>${node.parent.ctype} *</api:type> - <api:name>self</api:name> - </api:arg> -% endif -% for arg in node.parameters: -% if arg.type.ctype == '<varargs>': - <api:varargs/> -% else: - <api:arg> - <api:type>${formatter.format_type(arg.type) | x}</api:type> - <api:name>${arg.argname}</api:name> - </api:arg> -% endif -% endfor - </api:function> - </info> - <title>${node.symbol}</title> -<synopsis><code mime="text/x-csrc"> -${node.retval.type.ctype} ${node.symbol} (\ -% if node.is_method: -${node.parent.ctype} *self\ -%endif -% if len(node.parameters) == 0: -% if not node.is_method: -void\ -%endif -); -% elif node.is_method: -, -% endif -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -% if ix != 0: -${' ' * (len(formatter.format_type(node.retval.type)) + len(node.symbol) + 3)}\ -% endif -% if arg.type.ctype == '<varargs>': -...\ -% else: -${formatter.format_type(arg.type) | x} ${arg.argname}\ -% endif -% if ix == len(node.parameters) - 1: -); -% else: -, -%endif -% endfor -</code></synopsis> -${formatter.format(node.doc)} - -% if node.parameters or node.retval: -<table> -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -<tr> -<td><p>${arg.argname} :</p></td> -<td>${formatter.format(arg.doc)}</td> -</tr> -% endfor -% if node.retval: -<tr> -<td><p>Returns :</p></td> -<td>${formatter.format(node.retval.doc)}</td> -</tr> -% endif -</table> -% endif -% if node.version: -<p>Since ${node.version}</p> -% endif -</page> diff --git a/giscanner/mallard-C-namespace.tmpl b/giscanner/mallard-C-namespace.tmpl deleted file mode 100644 index 284ba238..00000000 --- a/giscanner/mallard-C-namespace.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -<?xml version="1.0"?> -<page id="index" - type="guide" - style="namespace" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - </info> - <title>${node.name} Documentation</title> - <links type="topic" ui:expanded="yes" groups="class" style="linklist"> - <title>Classes</title> - </links> - <links type="topic" ui:expanded="yes" groups="function" style="linklist"> - <title>Functions</title> - </links> - <links type="topic" ui:expanded="yes" groups="#first #default #last" style="linklist"> - <title>Other</title> - </links> -</page> diff --git a/giscanner/mallard-C-property.tmpl b/giscanner/mallard-C-property.tmpl deleted file mode 100644 index 2d37ba10..00000000 --- a/giscanner/mallard-C-property.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -<?xml version="1.0"?> -<page id="${namespace.name}.${node.name}" - type="topic" - style="property" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="property"/> - <title type="link" role="topic">${node.name}</title> - </info> - <title>${node.parent.ctype}:${node.name}</title> -${formatter.format(node.doc)} -</page> diff --git a/giscanner/mallard-C-record.tmpl b/giscanner/mallard-C-record.tmpl deleted file mode 100644 index a173e77a..00000000 --- a/giscanner/mallard-C-record.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -<?xml version="1.0"?> -<page id="${node.namespace.name}.${node.name}" - type="guide" - style="record" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="index"/> - </info> - <title>${node.namespace.name}${node.name}</title> -${formatter.format(node.doc)} -</page> diff --git a/giscanner/mallard-C-signal.tmpl b/giscanner/mallard-C-signal.tmpl deleted file mode 100644 index 7aae3ae4..00000000 --- a/giscanner/mallard-C-signal.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -<?xml version="1.0"?> -<page id="${namespace.name}.${node.name}" - type="topic" - style="signal" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="signal"/> - <title type="link" role="topic">${node.name}</title> - </info> - <title>${node.parent.ctype}::${node.name}</title> -${formatter.format(node.doc)} -</page> diff --git a/giscanner/mallard-C-vfunc.tmpl b/giscanner/mallard-C-vfunc.tmpl deleted file mode 100644 index 5b5bbfb1..00000000 --- a/giscanner/mallard-C-vfunc.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -<?xml version="1.0"?> -<page id="${page_id}" - type="topic" - style="vfunc" - xmlns="http://projectmallard.org/1.0/" - xmlns:api="http://projectmallard.org/experimental/api/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="vfunc"/> - </info> - <title>${node.name}</title> -<synopsis><code mime="text/x-csrc"> -</code></synopsis> -${formatter.format(node.doc)} - -% if node.parameters or node.retval: -<table> -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -<tr> -<td><p>${arg.argname} :</p></td> -<td>${formatter.format(arg.doc)}</td> -</tr> -% endfor -% if node.retval: -<tr> -<td><p>Returns :</p></td> -<td>${formatter.format(node.retval.doc)}</td> -</tr> -% endif -</table> -% endif -% if node.version: -<p>Since ${node.version}</p> -% endif -</page> diff --git a/giscanner/mallard-Python-class.tmpl b/giscanner/mallard-Python-class.tmpl deleted file mode 100644 index 04e5fc72..00000000 --- a/giscanner/mallard-Python-class.tmpl +++ /dev/null @@ -1,66 +0,0 @@ -<?xml version="1.0"?> -<page id="${node.namespace.name}.${node.name}" - type="guide" - style="class" - xmlns="http://projectmallard.org/1.0/" - xmlns:api="http://projectmallard.org/experimental/api/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="index" group="class"/> - </info> - <title>${namespace.name}.${node.name}</title> -${formatter.format(node.doc)} - - <synopsis><code> -from gi.repository import ${namespace.name} - -${formatter.to_underscores(node.name).lower()} = ${namespace.name}.${node.name}(\ -% for property_, ix in zip(node.properties, range(len(node.properties))): -% if property_.construct or property_.construct_only or property_.writable: -<link xref='${namespace.name}.${node.name}-${property_.name}'>${property_.name.replace('-', '_')}</link>=value\ -% if ix != len(node.properties) - 1: -, \ -% endif -% endif -% endfor -)\ - </code></synopsis> - -% if node.version: -<p>Since ${node.version}</p> -% endif - <synopsis> - <title>Hierarchy</title> - <tree> -% for class_ in formatter.get_class_hierarchy(node): - <item> - <code>${class_.namespace.name}.${class_.name}</code> -% endfor -% for class_ in formatter.get_class_hierarchy(node): - </item> -% endfor - </tree> - </synopsis> - <links type="topic" ui:expanded="yes" - api:type="function" api:mime="text/x-python" - groups="method" style="linklist"> - <title>Methods</title> - </links> - <links type="topic" ui:expanded="yes" - api:type="function" api:mime="text/x-python" - groups="function" style="linklist"> - <title>Functions</title> - </links> - <links type="topic" ui:expanded="yes" groups="property" style="linklist"> - <title>Properties</title> - </links> - <links type="topic" ui:expanded="yes" groups="signal" style="linklist"> - <title>Signals</title> - </links> - <links type="topic" ui:expanded="yes" groups="vfunc" style="linklist"> - <title>Virtual functions</title> - </links> - <links type="topic" ui:expanded="yes" groups="#first #default #last" style="linklist"> - <title>Other</title> - </links> -</page> diff --git a/giscanner/mallard-Python-default.tmpl b/giscanner/mallard-Python-default.tmpl deleted file mode 100644 index 683adf6a..00000000 --- a/giscanner/mallard-Python-default.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -<?xml version="1.0"?> -<page id="${page_id}" - type="topic" - style="" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - </info> - <title>${namespace.name}.${node.name}</title> -${formatter.format(node.doc)} -</page> diff --git a/giscanner/mallard-Python-enum.tmpl b/giscanner/mallard-Python-enum.tmpl deleted file mode 100644 index fd6ca0fb..00000000 --- a/giscanner/mallard-Python-enum.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0"?> -<page id="${node.namespace.name}.${node.name}" - type="guide" - style="enum" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="index"/> - </info> - <title>${node.namespace.name}.${node.name}</title> - ${formatter.format(node.doc)} -% if node.members: -<table> -% for member, ix in zip(node.members, range(len(node.members))): -<tr> -<td><p>${node.name}.${member.name.upper()} :</p></td> -<td>${formatter.format(member.doc)}</td> -</tr> -% endfor -</table> -% endif - -</page> diff --git a/giscanner/mallard-Python-function.tmpl b/giscanner/mallard-Python-function.tmpl deleted file mode 100644 index 7aa25e8e..00000000 --- a/giscanner/mallard-Python-function.tmpl +++ /dev/null @@ -1,88 +0,0 @@ -<?xml version="1.0"?> -<% -page_style = 'function' -if node.is_constructor: - page_style = 'constructor' -elif node.is_method: - page_style = 'method' -%> -<page id="${page_id}" - type="topic" - style="${page_style}" - xmlns="http://projectmallard.org/1.0/" - xmlns:api="http://projectmallard.org/experimental/api/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> -% if node.parent is not None: - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="${page_style}"/> -% else: - <link type="guide" xref="index" group="${page_style}"/> -% endif - <api:function> - <api:returns> - <api:type>${formatter.format_type(node.retval.type) | x}</api:type> - </api:returns> - <api:name>${node.symbol}</api:name> -% if node.is_method: - <api:arg> - <api:type>${node.parent.ctype} *</api:type> - <api:name>self</api:name> - </api:arg> -% endif -% for arg in node.parameters: -% if arg.type.ctype == '<varargs>': - <api:varargs/> -% else: - <api:arg> - <api:type>${formatter.format_type(arg.type) | x}</api:type> - <api:name>${arg.argname}</api:name> - </api:arg> -% endif -% endfor - </api:function> - </info> - <title>${node.name}</title> -<synopsis><code mime="text/x-python"> -% if len(node.parameters) != 0: -@accepts(\ -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -${formatter.format_type(arg.type) | x}\ -% if ix != len(node.parameters) - 1: -, \ -%endif -% endfor -) -% endif -@returns(${formatter.format_type(node.retval.type) | x}) -def \ -${node.name}(\ -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -${arg.argname}\ -% if ix != len(node.parameters) - 1: -, \ -%endif -% endfor -) -</code></synopsis> -${formatter.format(node.doc)} - -% if node.parameters or node.retval: -<table> -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -<tr> -<td><p>${arg.argname} :</p></td> -<td>${formatter.format(arg.doc)}</td> -</tr> -% endfor -% if node.retval and node.retval.type.ctype != 'void': -<tr> -<td><p>Returns :</p></td> -<td>${formatter.format(node.retval.doc)}</td> -</tr> -% endif -</table> -% endif -% if node.version: -<p>Since ${node.version}</p> -% endif -</page> diff --git a/giscanner/mallard-Python-namespace.tmpl b/giscanner/mallard-Python-namespace.tmpl deleted file mode 100644 index 935cd440..00000000 --- a/giscanner/mallard-Python-namespace.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -<?xml version="1.0"?> -<page id="index" - type="guide" - style="namespace" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - </info> - <title>${node.name} Documentation</title> - <links type="topic" ui:expanded="yes" groups="class"> - <title>Classes</title> - </links> - <links type="topic" ui:expanded="yes" groups="function"> - <title>Functions</title> - </links> - <links type="topic" ui:expanded="yes" groups="#first #default #last"> - <title>Other</title> - </links> -</page> diff --git a/giscanner/mallard-Python-property.tmpl b/giscanner/mallard-Python-property.tmpl deleted file mode 100644 index c4d2229e..00000000 --- a/giscanner/mallard-Python-property.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -<?xml version="1.0"?> -<page id="${namespace.name}.${node.parent.name}-${node.name}" - type="topic" - style="property" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="property"/> - <title type="link" role="topic">${node.name}</title> - </info> - <title>${namespace.name}.${node.parent.name}:${node.name}</title> -<synopsis><code mime="text/x-python"> -"${node.name}" ${formatter.format_type(node.type)} : ${formatter.format_property_flags(node)} -</code></synopsis> -${formatter.format(node.doc)} -</page> diff --git a/giscanner/mallard-Python-record.tmpl b/giscanner/mallard-Python-record.tmpl deleted file mode 100644 index 1b00e3be..00000000 --- a/giscanner/mallard-Python-record.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -<?xml version="1.0"?> -<page id="${node.namespace.name}.${node.name}" - type="guide" - style="record" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="index"/> - </info> - <title>${node.namespace.name}${node.name}</title> - <p>${node.doc}</p> -</page> diff --git a/giscanner/mallard-Python-signal.tmpl b/giscanner/mallard-Python-signal.tmpl deleted file mode 100644 index fed0659f..00000000 --- a/giscanner/mallard-Python-signal.tmpl +++ /dev/null @@ -1,52 +0,0 @@ -<?xml version="1.0"?> -<page id="${namespace.name}.${node.parent.name}-${node.name}" - type="topic" - style="signal" - xmlns="http://projectmallard.org/1.0/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="signal"/> - <title type="link" role="topic">${node.name}</title> - </info> - <title>${namespace.name}.${node.parent.name}::${node.name}</title> -<synopsis><code mime="text/x-python"> -def callback(${formatter.to_underscores(node.parent.name).lower()}, \ -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -${arg.argname}, \ -% endfor -user_param1, ...) -</code></synopsis> -${formatter.format(node.doc)} - -<table> -<tr> -<td><p>${formatter.to_underscores(node.parent.name).lower()} :</p></td> -<td><p>instance of ${namespace.name}.${node.parent.name} that is emitting the signal</p></td> -</tr> -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -<tr> -<td><p>${arg.argname} :</p></td> -<td>${formatter.format(arg.doc)}</td> -</tr> -% endfor -<tr> -<td><p>user_param1 :</p></td> -<td><p>first user parameter (if any) specified with the connect() method</p></td> -</tr> -<tr> -<td><p>... :</p></td> -<td><p>additional user parameters (if any)</p></td> -</tr> -% if node.retval and \ - node.retval.type.ctype != 'void' and \ - node.retval.type.ctype is not None: -<tr> -<td><p>Returns :</p></td> -<td>${node.retval.type.ctype} ${formatter.format(node.retval.doc)}</td> -</tr> -% endif -</table> -% if node.version: -<p>Since ${node.version}</p> -% endif -</page> diff --git a/giscanner/mallard-Python-vfunc.tmpl b/giscanner/mallard-Python-vfunc.tmpl deleted file mode 100644 index 7e95bc85..00000000 --- a/giscanner/mallard-Python-vfunc.tmpl +++ /dev/null @@ -1,56 +0,0 @@ -<?xml version="1.0"?> -<page id="${page_id}" - type="topic" - style="vfunc" - xmlns="http://projectmallard.org/1.0/" - xmlns:api="http://projectmallard.org/experimental/api/" - xmlns:ui="http://projectmallard.org/experimental/ui/"> - <info> - <link type="guide" xref="${namespace.name}.${node.parent.name}" group="vfunc"/> - <title type="link" role="topic">${node.name}</title> - </info> - <title>${namespace.name}.${node.parent.name}.${node.name}</title> -<synopsis><code mime="text/x-python"> -% if len(node.parameters) != 0: -@accepts(\ -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -${formatter.format_type(arg.type) | x}\ -% if ix != len(node.parameters) - 1: -, \ -%endif -% endfor -) -% endif -@returns(${formatter.format_type(node.retval.type) | x}) -def \ -do_${node.name}(self, \ -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -${arg.argname}\ -% if ix != len(node.parameters) - 1: -, \ -%endif -% endfor -): -</code></synopsis> -${formatter.format(node.doc)} - -% if node.parameters or node.retval: -<table> -% for arg, ix in zip(node.parameters, range(len(node.parameters))): -<tr> -<td><p>${arg.argname} :</p></td> -<td>${formatter.format(arg.doc)}</td> -</tr> -% endfor -% if node.retval and node.retval.type.ctype != 'void': -<tr> -<td><p>Returns :</p></td> -<td>${formatter.format(node.retval.doc)}</td> -</tr> -% endif -</table> -% endif -% if node.version: -<p>Since ${node.version}</p> -% endif -</page> diff --git a/giscanner/mallardwriter.py b/giscanner/mallardwriter.py deleted file mode 100644 index 9c833843..00000000 --- a/giscanner/mallardwriter.py +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env python -# -*- Mode: Python -*- -# GObject-Introspection - a framework for introspecting GObject libraries -# Copyright (C) 2010 Zach Goldberg -# Copyright (C) 2011 Johan Dahlin -# Copyright (C) 2011 Shaun McCance -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -# 02110-1301, USA. -# - -import os -import re -import tempfile - -from xml.sax import saxutils -from mako.template import Template - -from . import ast -from .utils import to_underscores - -def make_page_id(namespace, node): - if isinstance(node, ast.Namespace): - return 'index' - elif isinstance(node, (ast.Class, ast.Interface)): - return '%s.%s' % (namespace.name, node.name) - elif isinstance(node, ast.Record): - return '%s.%s' % (namespace.name, node.name) - elif isinstance(node, ast.Function): - if node.parent is not None: - return '%s.%s.%s' % (namespace.name, node.parent.name, node.name) - else: - return '%s.%s' % (namespace.name, node.name) - elif isinstance(node, ast.Enum): - return '%s.%s' % (namespace.name, node.name) - elif isinstance(node, ast.Property) and node.parent is not None: - return '%s.%s-%s' % (namespace.name, node.parent.name, node.name) - elif isinstance(node, ast.Signal) and node.parent is not None: - return '%s.%s-%s' % (namespace.name, node.parent.name, node.name) - elif isinstance(node, ast.VFunction) and node.parent is not None: - return '%s.%s-%s' % (namespace.name, node.parent.name, node.name) - else: - return '%s.%s' % (namespace.name, node.name) - -def make_template_name(node, language): - if isinstance(node, ast.Namespace): - node_kind = 'namespace' - elif isinstance(node, (ast.Class, ast.Interface)): - node_kind = 'class' - elif isinstance(node, ast.Record): - node_kind = 'record' - elif isinstance(node, ast.Function): - node_kind = 'function' - elif isinstance(node, ast.Enum): - node_kind = 'enum' - elif isinstance(node, ast.Property) and node.parent is not None: - node_kind = 'property' - elif isinstance(node, ast.Signal) and node.parent is not None: - node_kind = 'signal' - elif isinstance(node, ast.VFunction) and node.parent is not None: - node_kind = 'vfunc' - else: - node_kind = 'default' - - return 'mallard-%s-%s.tmpl' % (language, node_kind) - -class TemplatedScanner(object): - def __init__(self, specs): - self.specs = self.unmangle_specs(specs) - self.regex = self.make_regex(self.specs) - - def unmangle_specs(self, specs): - mangled = re.compile('<<([a-zA-Z_:]+)>>') - specdict = dict((name.lstrip('!'), spec) for name, spec in specs) - - def unmangle(spec, name=None): - def replace_func(match): - child_spec_name = match.group(1) - - if ':' in child_spec_name: - pattern_name, child_spec_name = child_spec_name.split(':', 1) - else: - pattern_name = None - - child_spec = specdict[child_spec_name] - # Force all child specs of this one to be unnamed - unmangled = unmangle(child_spec, None) - if pattern_name and name: - return '(?P<%s_%s>%s)' % (name, pattern_name, unmangled) - else: - return unmangled - - return mangled.sub(replace_func, spec) - - return [(name, unmangle(spec, name)) for name, spec in specs] - - def make_regex(self, specs): - regex = '|'.join('(?P<%s>%s)' % (name, spec) for name, spec in specs - if not name.startswith('!')) - return re.compile(regex) - - def get_properties(self, name, match): - groupdict = match.groupdict() - properties = {name: groupdict.pop(name)} - name = name + "_" - for group, value in groupdict.iteritems(): - if group.startswith(name): - key = group[len(name):] - properties[key] = value - return properties - - def scan(self, text): - pos = 0 - while True: - match = self.regex.search(text, pos) - if match is None: - break - - start = match.start() - if start > pos: - yield ('other', text[pos:start], None) - - pos = match.end() - name = match.lastgroup - yield (name, match.group(0), self.get_properties(name, match)) - - if pos < len(text): - yield ('other', text[pos:], None) - -class DocstringScanner(TemplatedScanner): - def __init__(self): - specs = [ - ('!alpha', r'[a-zA-Z0-9_]+'), - ('!alpha_dash', r'[a-zA-Z0-9_-]+'), - ('property', r'#<<type_name:alpha>>:(<<property_name:alpha_dash>>)'), - ('signal', r'#<<type_name:alpha>>::(<<signal_name:alpha_dash>>)'), - ('type_name', r'#(<<type_name:alpha>>)'), - ('fundamental', r'%(<<fundamental:alpha>>)'), - ('function_call', r'<<symbol_name:alpha>>\(\)'), - ] - - super(DocstringScanner, self).__init__(specs) - -class MallardFormatter(object): - def __init__(self, transformer): - self._transformer = transformer - self._scanner = DocstringScanner() - - def escape(self, text): - return saxutils.escape(text) - - def format(self, doc): - if doc is None: - return '' - - result = '' - for para in doc.split('\n\n'): - result += '<p>' - result += self.format_inline(para) - result += '</p>' - return result - - def _process_other(self, namespace, match, props): - return self.escape(match) - - def _find_thing(self, list_, name): - for item in list_: - if item.name == name: - return item - raise KeyError("Could not find %s" % (name, )) - - def _process_property(self, namespace, match, props): - type_node = namespace.get_by_ctype(props['type_name']) - if type_node is None: - return match - - try: - node = self._find_thing(type_node.properties, props['property_name']) - except (AttributeError, KeyError), e: - return match - - xref_name = "%s.%s:%s" % (namespace.name, type_node.name, node.name) - return '<link xref="%s">%s</link>' % (make_page_id(namespace, node), xref_name) - - def _process_signal(self, namespace, match, props): - type_node = namespace.get_by_ctype(props['type_name']) - if type_node is None: - return match - - try: - node = self._find_thing(type_node.signals, props['signal_name']) - except (AttributeError, KeyError), e: - return match - - xref_name = "%s.%s::%s" % (namespace.name, type_node.name, node.name) - return '<link xref="%s">%s</link>' % (make_page_id(namespace, node), xref_name) - - def _process_type_name(self, namespace, match, props): - node = namespace.get_by_ctype(props['type_name']) - if node is None: - return match - xref_name = "%s.%s" % (namespace.name, node.name) - return '<link xref="%s">%s</link>' % (make_page_id(namespace, node), xref_name) - - def _process_function_call(self, namespace, match, props): - node = namespace.get_by_symbol(props['symbol_name']) - if node is None: - return match - - return '<link xref="%s">%s</link>' % (make_page_id(namespace, node), - self.format_function_name(node)) - - def _process_fundamental(self, namespace, match, props): - return self.fundamentals.get(props['fundamental'], match) - - def _process_token(self, tok): - namespace = self._transformer.namespace - - kind, match, props = tok - - dispatch = { - 'other': self._process_other, - 'property': self._process_property, - 'signal': self._process_signal, - 'type_name': self._process_type_name, - 'function_call': self._process_function_call, - 'fundamental': self._process_fundamental, - } - - return dispatch[kind](namespace, match, props) - - def format_inline(self, para): - tokens = self._scanner.scan(para) - words = [self._process_token(tok) for tok in tokens] - return ''.join(words) - - def format_function_name(self, func): - raise NotImplementedError - - def format_type(self, type_): - raise NotImplementedError - - def format_property_flags(self, property_): - flags = [] - if property_.readable: - flags.append("Read") - if property_.writable: - flags.append("Write") - if property_.construct: - flags.append("Construct") - if property_.construct_only: - flags.append("Construct Only") - - return " / ".join(flags) - - def to_underscores(self, string): - return to_underscores(string) - - def get_class_hierarchy(self, node): - parent_chain = [node] - - while node.parent: - node = self._transformer.lookup_giname(str(node.parent)) - parent_chain.append(node) - - parent_chain.reverse() - return parent_chain - -class MallardFormatterC(MallardFormatter): - language = "C" - - fundamentals = { - "TRUE": "TRUE", - "FALSE": "FALSE", - "NULL": "NULL", - } - - def format_type(self, type_): - if isinstance(type_, ast.Array): - return self.format_type(type_.element_type) + '*' - elif type_.ctype is not None: - return type_.ctype - else: - return type_.target_fundamental - - def format_function_name(self, func): - return func.symbol - -class MallardFormatterPython(MallardFormatter): - language = "Python" - - fundamentals = { - "TRUE": "True", - "FALSE": "False", - "NULL": "None", - } - - def format_type(self, type_): - if isinstance(type_, ast.Array): - return '[' + self.format_type(type_.element_type) + ']' - elif isinstance(type_, ast.Map): - return '{%s: %s}' % (self.format_type(type_.key_type), - self.format_type(type_.value_type)) - elif type_.target_giname is not None: - return type_.target_giname - else: - return type_.target_fundamental - - def format_function_name(self, func): - if func.parent is not None: - return "%s.%s" % (func.parent.name, func.name) - else: - return func.name - -LANGUAGES = { - "c": MallardFormatterC, - "python": MallardFormatterPython, -} - -class MallardWriter(object): - def __init__(self, transformer, language): - self._transformer = transformer - - try: - formatter_class = LANGUAGES[language.lower()] - except KeyError: - raise SystemExit("Unsupported language: %s" % (language, )) - - self._formatter = formatter_class(self._transformer) - self._language = self._formatter.language - - def write(self, output): - nodes = [self._transformer.namespace] - for node in self._transformer.namespace.itervalues(): - if isinstance(node, ast.Function) and node.moved_to is not None: - continue - if getattr(node, 'disguised', False): - continue - if isinstance(node, ast.Record) and \ - self._language == 'Python' and \ - node.is_gtype_struct_for is not None: - continue - nodes.append(node) - if isinstance(node, (ast.Class, ast.Interface, ast.Record)): - nodes += getattr(node, 'methods', []) - nodes += getattr(node, 'static_methods', []) - nodes += getattr(node, 'virtual_methods', []) - nodes += getattr(node, 'properties', []) - nodes += getattr(node, 'signals', []) - if self._language == 'C': - nodes += getattr(node, 'constructors', []) - for node in nodes: - self._render_node(node, output) - - def _render_node(self, node, output): - namespace = self._transformer.namespace - - if 'UNINSTALLED_INTROSPECTION_SRCDIR' in os.environ: - top_srcdir = os.environ['UNINSTALLED_INTROSPECTION_SRCDIR'] - template_dir = os.path.join(top_srcdir, 'giscanner') - else: - template_dir = os.path.dirname(__file__) - - template_name = make_template_name(node, self._language) - page_id = make_page_id(namespace, node) - - file_name = os.path.join(template_dir, template_name) - file_name = os.path.abspath(file_name) - template = Template(filename=file_name, output_encoding='utf-8', - module_directory=tempfile.gettempdir()) - result = template.render(namespace=namespace, - node=node, - page_id=page_id, - formatter=self._formatter) - - output_file_name = os.path.join(os.path.abspath(output), - page_id + '.page') - fp = open(output_file_name, 'w') - fp.write(result) - fp.close() diff --git a/giscanner/message.py b/giscanner/message.py index 8a948cd3..1a0a6c52 100644 --- a/giscanner/message.py +++ b/giscanner/message.py @@ -31,9 +31,13 @@ from . import utils class Position(object): - """Represents a position in the source file which we + """ + Represents a position in the source file which we want to inform about. """ + + __slots__ = ('filename', 'line', 'column') + def __init__(self, filename=None, line=None, column=None): self.filename = filename self.line = line @@ -44,15 +48,16 @@ class Position(object): (other.filename, other.line, other.column)) def __repr__(self): - return '<Position %s:%d:%d>' % ( - os.path.basename(self.filename), - self.line or -1, - self.column or -1) + return '<Position %s:%d:%d>' % (os.path.basename(self.filename), self.line or -1, + self.column or -1) def format(self, cwd): - filename = self.filename - if filename.startswith(cwd): - filename = filename[len(cwd):] + filename = os.path.realpath(self.filename) + cwd = os.path.realpath(cwd) + common_prefix = os.path.commonprefix((filename, cwd)) + if common_prefix: + filename = os.path.relpath(filename, common_prefix) + if self.column is not None: return '%s:%d:%d' % (filename, self.line, self.column) elif self.line is not None: @@ -60,9 +65,6 @@ class Position(object): else: return '%s:' % (filename, ) - def offset(self, offset): - return Position(self.filename, self.line+offset, self.column) - class MessageLogger(object): _instance = None @@ -70,11 +72,12 @@ class MessageLogger(object): def __init__(self, namespace, output=None): if output is None: output = sys.stderr - self._cwd = os.getcwd() + os.sep + self._cwd = os.getcwd() self._output = output self._namespace = namespace - self._enable_warnings = False + self._enable_warnings = [] self._warning_count = 0 + self._error_count = 0 @classmethod def get(cls, *args, **kwargs): @@ -82,24 +85,27 @@ class MessageLogger(object): cls._instance = cls(*args, **kwargs) return cls._instance - def enable_warnings(self, enable): - self._enable_warnings = enable + def enable_warnings(self, log_types): + self._enable_warnings = log_types def get_warning_count(self): return self._warning_count + def get_error_count(self): + return self._error_count + def log(self, log_type, text, positions=None, prefix=None): - """Log a warning, using optional file positioning information. -If the warning is related to a ast.Node type, see log_node().""" + """ + Log a warning, using optional file positioning information. + If the warning is related to a ast.Node type, see log_node(). + """ utils.break_on_debug_flag('warning') self._warning_count += 1 - if not self._enable_warnings and log_type != FATAL: + if not log_type in self._enable_warnings: return - # Always drop through on fatal - if type(positions) == set: positions = list(positions) if isinstance(positions, Position): @@ -116,31 +122,34 @@ If the warning is related to a ast.Node type, see log_node().""" error_type = "Warning" elif log_type == ERROR: error_type = "Error" + self._error_count += 1 elif log_type == FATAL: error_type = "Fatal" + if prefix: - text = ( -'''%s: %s: %s: %s: %s\n''' % (last_position, error_type, self._namespace.name, - prefix, text)) + text = ('%s: %s: %s: %s: %s\n' % (last_position, error_type, + self._namespace.name, prefix, text)) else: if self._namespace: - text = ( -'''%s: %s: %s: %s\n''' % (last_position, error_type, self._namespace.name, text)) + text = ('%s: %s: %s: %s\n' % (last_position, error_type, + self._namespace.name, text)) else: - text = ( -'''%s: %s: %s\n''' % (last_position, error_type, text)) + text = ('%s: %s: %s\n' % (last_position, error_type, text)) self._output.write(text) + if log_type == FATAL: utils.break_on_debug_flag('fatal') raise SystemExit(text) def log_node(self, log_type, node, text, context=None, positions=None): - """Log a warning, using information about file positions from -the given node. The optional context argument, if given, should be -another ast.Node type which will also be displayed. If no file position -information is available from the node, the position data from the -context will be used.""" + """ + Log a warning, using information about file positions from + the given node. The optional context argument, if given, should be + another ast.Node type which will also be displayed. If no file position + information is available from the node, the position data from the + context will be used. + """ if positions: pass elif getattr(node, 'file_positions', None): @@ -169,17 +178,26 @@ def log_node(log_type, node, text, context=None, positions=None): ml = MessageLogger.get() ml.log_node(log_type, node, text, context=context, positions=positions) + def warn(text, positions=None, prefix=None): ml = MessageLogger.get() ml.log(WARNING, text, positions, prefix) + def warn_node(node, text, context=None, positions=None): log_node(WARNING, node, text, context=context, positions=positions) + def warn_symbol(symbol, text): ml = MessageLogger.get() ml.log_symbol(WARNING, symbol, text) + +def error(text, positions=None, prefix=None): + ml = MessageLogger.get() + ml.log(ERROR, text, positions, prefix) + + def fatal(text, positions=None, prefix=None): ml = MessageLogger.get() ml.log(FATAL, text, positions, prefix) diff --git a/giscanner/odict.py b/giscanner/odict.py deleted file mode 100644 index df703cbb..00000000 --- a/giscanner/odict.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- Mode: Python -*- -# GObject-Introspection - a framework for introspecting GObject libraries -# Copyright (C) 2008 Johan Dahlin -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. -# - -"""odict - an ordered dictionary""" - -from UserDict import DictMixin - - -class odict(DictMixin): - - def __init__(self): - self._items = {} - self._keys = [] - - def __setitem__(self, key, value): - if key not in self._items: - self._keys.append(key) - self._items[key] = value - - def __getitem__(self, key): - return self._items[key] - - def __delitem__(self, key): - del self._items[key] - self._keys.remove(key) - - def keys(self): - return self._keys[:] diff --git a/giscanner/scannerlexer.l b/giscanner/scannerlexer.l index a783ec06..42f85eab 100644 --- a/giscanner/scannerlexer.l +++ b/giscanner/scannerlexer.l @@ -31,11 +31,17 @@ %{ #include <ctype.h> #include <stdio.h> +#ifndef _WIN32 +#include <limits.h> +#endif #include <glib.h> #include "sourcescanner.h" #include "scannerparser.h" -#include "grealpath.h" + +#ifdef USE_WINDOWS +#include <windows.h> +#endif int lineno; char linebuf[2000]; @@ -51,6 +57,7 @@ static void parse_trigraph (GISourceScanner *scanner); static void process_linemarks (GISourceScanner *scanner); static int check_identifier (GISourceScanner *scanner, const char *); static int parse_ignored_macro (void); +static void print_error (GISourceScanner *scanner); %} %option nounput @@ -71,6 +78,7 @@ stringtext ([^\\\"])|(\\.) ++lineno; } "\\\n" { ++lineno; } + [\t\f\v\r ]+ { /* Ignore whitespace. */ } "/*" { parse_comment(scanner); } @@ -79,6 +87,14 @@ stringtext ([^\\\"])|(\\.) "#define "[a-zA-Z_][a-zA-Z_0-9]*"(" { yyless (yyleng - 1); return FUNCTION_MACRO; } "#define "[a-zA-Z_][a-zA-Z_0-9]* { return OBJECT_MACRO; } +"#ifdef"[\t ]+"__GI_SCANNER__"[\t ]?.*"\n" { return IFDEF_GI_SCANNER; } +"#ifndef"[\t ]+"__GI_SCANNER__"[\t ]?.*"\n" { return IFNDEF_GI_SCANNER; } +"#ifndef ".*"\n" { return IFNDEF_COND; } +"#ifdef ".*"\n" { return IFDEF_COND; } +"#if ".*"\n" { return IF_COND; } +"#elif ".*"\n" { return ELIF_COND; } +"#else".*"\n" { return ELSE_COND; } +"#endif".*"\n" { return ENDIF_COND; } "#pragma ".*"\n" { /* Ignore pragma. */ } "# "[0-9]+" ".*"\n" { process_linemarks(scanner); } @@ -134,21 +150,32 @@ stringtext ([^\\\"])|(\\.) "," { return ','; } "->" { return ARROW; } +"__asm"[\t\f\v\r ]+"volatile" { if (!parse_ignored_macro()) REJECT; } +"__asm__"[\t\f\v\r ]+"volatile" { if (!parse_ignored_macro()) REJECT; } "__asm" { if (!parse_ignored_macro()) REJECT; } "__asm__" { if (!parse_ignored_macro()) REJECT; } "__attribute__" { if (!parse_ignored_macro()) REJECT; } "__attribute" { if (!parse_ignored_macro()) REJECT; } "__const" { return CONST; } "__extension__" { return EXTENSION; } +"__inline__" { return INLINE; } "__inline" { return INLINE; } "__nonnull" { if (!parse_ignored_macro()) REJECT; } "__signed__" { return SIGNED; } "__restrict" { return RESTRICT; } "__typeof" { if (!parse_ignored_macro()) REJECT; } +"__volatile" { return VOLATILE; } +"__volatile__" { return VOLATILE; } "_Bool" { return BOOL; } "G_GINT64_CONSTANT" { return INTL_CONST; } "G_GUINT64_CONSTANT" { return INTUL_CONST; } + +"TRUE" { return BOOLEAN; } +"FALSE" { return BOOLEAN; } +"true" { return BOOLEAN; } +"false" { return BOOLEAN; } + [a-zA-Z_][a-zA-Z_0-9]* { if (scanner->macro_scan) return check_identifier(scanner, yytext); else REJECT; } "asm" { if (!parse_ignored_macro()) REJECT; } @@ -170,6 +197,10 @@ stringtext ([^\\\"])|(\\.) "if" { return IF; } "inline" { return INLINE; } "int" { return INT; } +"__uint128_t" { return INT; } +"__int128_t" { return INT; } +"__uint128" { return INT; } +"__int128" { return INT; } "long" { return LONG; } "register" { return REGISTER; } "restrict" { return RESTRICT; } @@ -202,7 +233,7 @@ stringtext ([^\\\"])|(\\.) "\""{stringtext}*"\"" { return STRING; } "L\""{stringtext}*"\"" { return STRING; } -. { if (yytext[0]) fprintf(stderr, "%s:%d: unexpected character `%c'\n", scanner->current_filename, lineno, yytext[0]); } +. { print_error(scanner); } %% @@ -212,54 +243,70 @@ yywrap (void) return 1; } - static void parse_comment (GISourceScanner *scanner) { - GString *string = NULL; int c1, c2; + GString *string = NULL; GISourceComment *comment; int comment_lineno; int skip = FALSE; - if (!g_list_find_custom (scanner->filenames, - scanner->current_filename, - (GCompareFunc)g_strcmp0)) { - skip = TRUE; - } else { - string = g_string_new ("/*"); - } - c1 = input(); c2 = input(); - comment_lineno = lineno; + if (c2 != EOF && (c1 == '*' && c2 != '*' && c2 != '/')) { + /* + * Store GTK-Doc comment blocks, + * starts with one '/' followed by exactly two '*' and not followed by a '/' + */ + if (!g_hash_table_contains (scanner->files, scanner->current_file)) { + skip = TRUE; + } else { + string = g_string_new (yytext); + } - while (c2 != EOF && !(c1 == '*' && c2 == '/')) - { - if (!skip) - g_string_append_c (string, c1); + comment_lineno = lineno; - if (c1 == '\n') - lineno++; + while (c2 != EOF && !(c1 == '*' && c2 == '/')) + { + if (!skip) + g_string_append_c (string, c1); - c1 = c2; - c2 = input(); - } + if (c1 == '\n') + lineno++; - if (skip) { - return; - } + c1 = c2; + c2 = input(); + } + + if (skip) { + return; + } - g_string_append (string, "*/"); + g_string_append (string, "*/"); - comment = g_slice_new (GISourceComment); - comment->comment = g_string_free (string, FALSE); - comment->line = comment_lineno; - comment->filename = g_strdup(scanner->current_filename); + comment = g_slice_new (GISourceComment); + comment->comment = g_string_free (string, FALSE); + comment->line = comment_lineno; + comment->filename = g_file_get_parse_name (scanner->current_file); - scanner->comments = g_slist_prepend (scanner->comments, - comment); + gi_source_scanner_take_comment (scanner, comment); + } else { + /* + * Ignore all other comment blocks + */ + while (c2 != EOF && !(c1 == '*' && c2 == '/')) + { + if (c1 == '\n') + lineno++; + + c1 = c2; + c2 = input(); + } + + return; + } } static int @@ -280,6 +327,58 @@ check_identifier (GISourceScanner *scanner, return IDENTIFIER; } +/* taken from glib/gfileutils.c */ +#if defined(MAXPATHLEN) +#define G_PATH_LENGTH MAXPATHLEN +#elif defined(PATH_MAX) +#define G_PATH_LENGTH PATH_MAX +#elif defined(_PC_PATH_MAX) +#define G_PATH_LENGTH sysconf(_PC_PATH_MAX) +#else +#define G_PATH_LENGTH 2048 +#endif + +static inline char * +_realpath (const char *path) +{ +#ifndef _WIN32 + char buffer[G_PATH_LENGTH]; + + return realpath (path, buffer) ? g_strdup (buffer) : NULL; +#else + /* We don't want to include <windows.h> as it clashes horribly + * with token names from scannerparser.h. So just declare + * GetFullPathNameA() here unless we already defined it, like + * in giscanner.c. + */ +#ifndef USE_WINDOWS + extern __stdcall GetFullPathNameA(const char*, int, char*, char**); +#endif + char *buffer; + char dummy; + int rc, len; + + rc = GetFullPathNameA (path, 1, &dummy, NULL); + if (rc == 0) + { + /* Weird failure, so just return the input path as such */ + return g_strdup (path); + } + + len = rc + 1; + buffer = g_malloc (len); + + rc = GetFullPathNameA (path, len, buffer, NULL); + if (rc == 0 || rc > len) + { + /* Weird failure again */ + g_free (buffer); + return g_strdup (path); + } + return buffer; +#endif +} + /* * # linenum "filename" flags * See http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html @@ -288,21 +387,24 @@ check_identifier (GISourceScanner *scanner, static void process_linemarks (GISourceScanner *scanner) { - char filename[1025]; - char *compress; + char escaped_filename[1025]; + char *filename; char *real; - sscanf(yytext, "# %d \"%1024[^\"]\"", &lineno, filename); - - compress = g_strcompress (filename); - real = g_realpath (filename); - if (real) { - g_free (scanner->current_filename); - scanner->current_filename = real; - } else { - g_free (real); - } - g_free (compress); + sscanf(yytext, "# %d \"%1024[^\"]\"", &lineno, escaped_filename); + filename = g_strcompress (escaped_filename); + + real = _realpath (filename); + if (real) + { + g_free (filename); + filename = real; + } + + if (scanner->current_file) + g_object_unref (scanner->current_file); + scanner->current_file = g_file_new_for_path (filename); + g_free (filename); } /* @@ -372,3 +474,13 @@ parse_trigraph (GISourceScanner *scanner) } g_strfreev (items); } + +static void +print_error (GISourceScanner *scanner) +{ + if (yytext[0]) { + char *filename = g_file_get_parse_name (scanner->current_file); + fprintf(stderr, "%s:%d: unexpected character `%c'\n", filename, lineno, yytext[0]); + g_free (filename); + } +} diff --git a/giscanner/scannermain.py b/giscanner/scannermain.py index 794cede3..98d56878 100755 --- a/giscanner/scannermain.py +++ b/giscanner/scannermain.py @@ -27,9 +27,10 @@ import shutil import subprocess import sys import tempfile +import platform from giscanner import message -from giscanner.annotationparser import AnnotationParser +from giscanner.annotationparser import GtkDocCommentBlockParser from giscanner.ast import Include, Namespace from giscanner.dumper import compile_introspection_binary from giscanner.gdumpparser import GDumpParser, IntrospectionBinary @@ -38,15 +39,42 @@ from giscanner.girparser import GIRParser from giscanner.girwriter import GIRWriter from giscanner.maintransformer import MainTransformer from giscanner.shlibs import resolve_shlibs -from giscanner.sourcescanner import SourceScanner +from giscanner.sourcescanner import SourceScanner, ALL_EXTS from giscanner.transformer import Transformer from . import utils + +def process_cflags_begin(option, opt, value, parser): + cflags = getattr(parser.values, option.dest) + while len(parser.rargs) > 0 and parser.rargs[0] != '--cflags-end': + arg = parser.rargs.pop(0) + if arg == "-I" and parser.rargs and parser.rargs[0] != '--cflags-end': + # This is a special case where there's a space between -I and the path. + arg += parser.rargs.pop(0) + cflags.append(utils.cflag_real_include_path(arg)) + + +def process_cflags_end(option, opt, value, parser): + pass + + +def process_cpp_includes(option, opt, value, parser): + cpp_includes = getattr(parser.values, option.dest) + cpp_includes.append(os.path.realpath(value)) + + def get_preprocessor_option_group(parser): group = optparse.OptionGroup(parser, "Preprocessor options") + group.add_option("", "--cflags-begin", + help="Start preprocessor/compiler flags", + dest="cflags", default=[], + action="callback", callback=process_cflags_begin) + group.add_option("", "--cflags-end", + help="End preprocessor/compiler flags", + action="callback", callback=process_cflags_end) group.add_option("-I", help="Pre-processor include file", - action="append", dest="cpp_includes", - default=[]) + dest="cpp_includes", default=[], type="string", + action="callback", callback=process_cpp_includes) group.add_option("-D", help="Pre-processor define", action="append", dest="cpp_defines", default=[]) @@ -56,6 +84,7 @@ def get_preprocessor_option_group(parser): group.add_option("-p", dest="", help="Ignored") return group + def get_windows_option_group(parser): group = optparse.OptionGroup(parser, "Machine Dependent Options") group.add_option("-m", help="some machine dependent option", @@ -64,13 +93,13 @@ def get_windows_option_group(parser): return group + def _get_option_parser(): parser = optparse.OptionParser('%prog [options] sources') parser.add_option('', "--quiet", action="store_true", dest="quiet", default=False, - help="If passed, do not print details of normal" \ - + " operation") + help="If passed, do not print details of normal operation") parser.add_option("", "--format", action="store", dest="format", default="gir", @@ -158,6 +187,9 @@ match the namespace prefix.""") parser.add_option("", "--c-include", action="append", dest="c_includes", default=[], help="headers which should be included in C programs") + parser.add_option("", "--filelist", + action="store", dest="filelist", default=[], + help="file containing headers and sources to be scanned") group = get_preprocessor_option_group(parser) parser.add_option_group(group) @@ -186,17 +218,15 @@ match the namespace prefix.""") def _error(msg): raise SystemExit('ERROR: %s' % (msg, )) + def passthrough_gir(path, f): parser = GIRParser() parser.parse(path) - writer = GIRWriter(parser.get_namespace(), - parser.get_shared_libraries(), - parser.get_includes(), - parser.get_pkgconfig_packages(), - parser.get_c_includes()) + writer = GIRWriter(parser.get_namespace()) f.write(writer.get_xml()) + def test_codegen(optstring): (namespace, out_h_filename, out_c_filename) = optstring.split(',') if namespace == 'Everything': @@ -207,6 +237,7 @@ def test_codegen(optstring): _error("Invaild namespace %r" % (namespace, )) return 0 + def process_options(output, allowed_flags): for option in output.split(): for flag in allowed_flags: @@ -215,6 +246,7 @@ def process_options(output, allowed_flags): yield option break + def process_packages(options, packages): args = ['pkg-config', '--cflags'] args.extend(packages) @@ -230,26 +262,47 @@ def process_packages(options, packages): filtered_output = list(process_options(output, options_whitelist)) parser = _get_option_parser() pkg_options, unused = parser.parse_args(filtered_output) - options.cpp_includes.extend(pkg_options.cpp_includes) + options.cpp_includes.extend([os.path.realpath(f) for f in pkg_options.cpp_includes]) options.cpp_defines.extend(pkg_options.cpp_defines) options.cpp_undefines.extend(pkg_options.cpp_undefines) + def extract_filenames(args): filenames = [] for arg in args: # We don't support real C++ parsing yet, but we should be able # to understand C API implemented in C++ files. - if (arg.endswith('.c') or arg.endswith('.cpp') or - arg.endswith('.cc') or arg.endswith('.cxx') or - arg.endswith('.h') or arg.endswith('.hpp') or - arg.endswith('.hxx')): + if os.path.splitext(arg)[1] in ALL_EXTS: if not os.path.exists(arg): _error('%s: no such a file or directory' % (arg, )) # Make absolute, because we do comparisons inside scannerparser.c # against the absolute path that cpp will give us - filenames.append(os.path.abspath(arg)) + filenames.append(arg) return filenames + +def extract_filelist(options): + filenames = [] + if not os.path.exists(options.filelist): + _error('%s: no such filelist file' % (options.filelist, )) + filelist_file = open(options.filelist, "r") + lines = filelist_file.readlines() + for line in lines: + # We don't support real C++ parsing yet, but we should be able + # to understand C API implemented in C++ files. + filename = line.strip() + if (filename.endswith('.c') or filename.endswith('.cpp') + or filename.endswith('.cc') or filename.endswith('.cxx') + or filename.endswith('.h') or filename.endswith('.hpp') + or filename.endswith('.hxx')): + if not os.path.exists(filename): + _error('%s: Invalid filelist entry-no such file or directory' % (line, )) + # Make absolute, because we do comparisons inside scannerparser.c + # against the absolute path that cpp will give us + filenames.append(filename) + return filenames + + def create_namespace(options): if options.strip_prefix: print """g-ir-scanner: warning: Option --strip-prefix has been deprecated; @@ -278,6 +331,7 @@ see --identifier-prefix and --symbol-prefix.""" identifier_prefixes=identifier_prefixes, symbol_prefixes=symbol_prefixes) + def create_transformer(namespace, options): transformer = Transformer(namespace, accept_unprefixed=options.accept_unprefixed) @@ -286,7 +340,6 @@ def create_transformer(namespace, options): transformer.disable_cache() transformer.set_passthrough_mode() - shown_include_warning = False for include in options.includes: if os.sep in include: _error("Invalid include path %r" % (include, )) @@ -300,6 +353,7 @@ def create_transformer(namespace, options): return transformer + def create_binary(transformer, options, args): # Transform the C AST nodes into higher level # GLib/GObject nodes @@ -310,7 +364,7 @@ def create_binary(transformer, options, args): gdump_parser.init_parse() if options.program: - args=[options.program] + args = [options.program] args.extend(options.program_args) binary = IntrospectionBinary(args) else: @@ -323,19 +377,28 @@ def create_binary(transformer, options, args): gdump_parser.parse() return shlibs + def create_source_scanner(options, args): - filenames = extract_filenames(args) + if hasattr(options, 'filelist') and options.filelist: + filenames = extract_filelist(options) + else: + filenames = extract_filenames(args) + + if platform.system() == 'Darwin': + options.cpp_undefines.append('__BLOCKS__') # Run the preprocessor, tokenize and construct simple # objects representing the raw C symbols ss = SourceScanner() ss.set_cpp_options(options.cpp_includes, options.cpp_defines, - options.cpp_undefines) + options.cpp_undefines, + cflags=options.cflags) ss.parse_files(filenames) ss.parse_macros(filenames) return ss + def write_output(data, options): if options.output == "-": output = sys.stdout @@ -355,7 +418,7 @@ def write_output(data, options): os.unlink(temp_f_name) try: shutil.move(main_f_name, options.output) - except OSError, e: + except OSError as e: if e.errno == errno.EPERM: os.unlink(main_f_name) return 0 @@ -364,14 +427,15 @@ def write_output(data, options): else: try: output = open(options.output, "w") - except IOError, e: + except IOError as e: _error("opening output for writing: %s" % (e.strerror, )) try: output.write(data) - except IOError, e: + except IOError as e: _error("while writing output: %s" % (e.strerror, )) + def scanner_main(args): parser = _get_option_parser() (options, args) = parser.parse_args(args) @@ -381,8 +445,9 @@ def scanner_main(args): if options.test_codegen: return test_codegen(options.test_codegen) - if len(args) <= 1: - _error('Need at least one filename') + if hasattr(options, 'filelist') and not options.filelist: + if len(args) <= 1: + _error('Need at least one filename') if not options.namespace_name: _error('Namespace name missing') @@ -400,7 +465,8 @@ def scanner_main(args): namespace = create_namespace(options) logger = message.MessageLogger.get(namespace=namespace) if options.warn_all: - logger.enable_warnings(True) + logger.enable_warnings((message.WARNING, message.ERROR, message.FATAL)) + transformer = create_transformer(namespace, options) packages = set(options.packages) @@ -412,11 +478,10 @@ def scanner_main(args): ss = create_source_scanner(options, args) - ap = AnnotationParser() - blocks = ap.parse(ss.get_comments()) + cbp = GtkDocCommentBlockParser() + blocks = cbp.parse_comment_blocks(ss.get_comments()) # Transform the C symbols into AST nodes - transformer.set_annotations(blocks) transformer.parse(ss.get_symbols()) if not options.header_only: @@ -424,6 +489,8 @@ def scanner_main(args): else: shlibs = [] + transformer.namespace.shared_libraries = shlibs + main = MainTransformer(transformer, blocks) main.transform() @@ -446,8 +513,9 @@ def scanner_main(args): else: exported_packages = options.packages - writer = Writer(transformer.namespace, shlibs, transformer.get_includes(), - exported_packages, options.c_includes) + transformer.namespace.c_includes = options.c_includes + transformer.namespace.exported_packages = exported_packages + writer = Writer(transformer.namespace) data = writer.get_xml() write_output(data, options) diff --git a/giscanner/scannerparser.y b/giscanner/scannerparser.y index 06a10efa..6cbf36ad 100644 --- a/giscanner/scannerparser.y +++ b/giscanner/scannerparser.y @@ -1,4 +1,4 @@ -/* GObject introspection: C parser +/* GObject introspection: C parser -*- indent-tabs-mode: t; tab-width: 8 -*- * * Copyright (c) 1997 Sandro Sigala <ssigala@globalnet.it> * Copyright (c) 2007-2008 Jürg Billeter <j@bitron.ch> @@ -128,6 +128,84 @@ out: return dest; } +enum { + IRRELEVANT = 1, + NOT_GI_SCANNER = 2, + FOR_GI_SCANNER = 3, +}; + +static void +update_skipping (GISourceScanner *scanner) +{ + GList *l; + for (l = scanner->conditionals.head; l != NULL; l = g_list_next (l)) + { + if (GPOINTER_TO_INT (l->data) == NOT_GI_SCANNER) + { + scanner->skipping = TRUE; + return; + } + } + + scanner->skipping = FALSE; +} + +static void +push_conditional (GISourceScanner *scanner, + gint type) +{ + g_assert (type != 0); + g_queue_push_head (&scanner->conditionals, GINT_TO_POINTER (type)); +} + +static gint +pop_conditional (GISourceScanner *scanner) +{ + gint type = GPOINTER_TO_INT (g_queue_pop_head (&scanner->conditionals)); + + if (type == 0) + { + gchar *filename = g_file_get_path (scanner->current_file); + fprintf (stderr, "%s:%d: mismatched %s", filename, lineno, yytext); + g_free (filename); + } + + return type; +} + +static void +warn_if_cond_has_gi_scanner (GISourceScanner *scanner, + const gchar *text) +{ + /* Some other conditional that is not __GI_SCANNER__ */ + if (strstr (text, "__GI_SCANNER__")) + { + gchar *filename = g_file_get_path (scanner->current_file); + fprintf (stderr, "%s:%d: the __GI_SCANNER__ constant should only be used with simple #ifdef or #endif: %s", + filename, lineno, text); + g_free (filename); + } +} + +static void +toggle_conditional (GISourceScanner *scanner) +{ + switch (pop_conditional (scanner)) + { + case FOR_GI_SCANNER: + push_conditional (scanner, NOT_GI_SCANNER); + break; + case NOT_GI_SCANNER: + push_conditional (scanner, FOR_GI_SCANNER); + break; + case 0: + break; + default: + push_conditional (scanner, IRRELEVANT); + break; + } +} + %} %error-verbose @@ -148,7 +226,7 @@ out: %token <str> IDENTIFIER "identifier" %token <str> TYPEDEF_NAME "typedef-name" -%token INTEGER FLOATING CHARACTER STRING +%token INTEGER FLOATING BOOLEAN CHARACTER STRING %token INTL_CONST INTUL_CONST %token ELLIPSIS ADDEQ SUBEQ MULEQ DIVEQ MODEQ XOREQ ANDEQ OREQ SL SR @@ -160,6 +238,8 @@ out: %token VOID VOLATILE WHILE %token FUNCTION_MACRO OBJECT_MACRO +%token IFDEF_GI_SCANNER IFNDEF_GI_SCANNER +%token IFDEF_COND IFNDEF_COND IF_COND ELIF_COND ELSE_COND ENDIF_COND %start translation_unit @@ -225,32 +305,42 @@ primary_expression { $$ = g_hash_table_lookup (const_table, $1); if ($$ == NULL) { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } else { $$ = gi_source_symbol_ref ($$); } } | INTEGER { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + char *rest; + guint64 value; + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; if (g_str_has_prefix (yytext, "0x") && strlen (yytext) > 2) { - $$->const_int = g_ascii_strtoll (yytext + 2, NULL, 16); + value = g_ascii_strtoull (yytext + 2, &rest, 16); } else if (g_str_has_prefix (yytext, "0") && strlen (yytext) > 1) { - $$->const_int = g_ascii_strtoll (yytext + 1, NULL, 8); + value = g_ascii_strtoull (yytext + 1, &rest, 8); } else { - $$->const_int = g_ascii_strtoll (yytext, NULL, 10); + value = g_ascii_strtoull (yytext, &rest, 10); } + $$->const_int = value; + $$->const_int_is_unsigned = (rest && (rest[0] == 'U')); + } + | BOOLEAN + { + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); + $$->const_boolean_set = TRUE; + $$->const_boolean = g_ascii_strcasecmp (yytext, "true") == 0 ? TRUE : FALSE; } | CHARACTER { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = g_utf8_get_char(yytext + 1); } | FLOATING { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_double_set = TRUE; $$->const_double = 0.0; sscanf (yytext, "%lf", &($$->const_double)); @@ -262,7 +352,7 @@ primary_expression } | EXTENSION '(' '{' block_item_list '}' ')' { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } ; @@ -270,7 +360,7 @@ primary_expression strings : STRING { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); yytext[strlen (yytext) - 1] = '\0'; $$->const_string = parse_c_string_literal (yytext + 1); if (!g_utf8_validate ($$->const_string, -1, NULL)) @@ -312,31 +402,31 @@ postfix_expression : primary_expression | postfix_expression '[' expression ']' { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | postfix_expression '(' argument_expression_list ')' { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | postfix_expression '(' ')' { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | postfix_expression '.' identifier_or_typedef_name { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | postfix_expression ARROW identifier_or_typedef_name { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | postfix_expression PLUSPLUS { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | postfix_expression MINUSMINUS { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } ; @@ -349,11 +439,11 @@ unary_expression : postfix_expression | PLUSPLUS unary_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | MINUSMINUS unary_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | unary_operator cast_expression { @@ -362,19 +452,19 @@ unary_expression $$ = $2; break; case UNARY_MINUS: - $$ = $2; + $$ = gi_source_symbol_copy ($2); $$->const_int = -$2->const_int; break; case UNARY_BITWISE_COMPLEMENT: - $$ = $2; + $$ = gi_source_symbol_copy ($2); $$->const_int = ~$2->const_int; break; case UNARY_LOGICAL_NEGATION: - $$ = $2; + $$ = gi_source_symbol_copy ($2); $$->const_int = !gi_source_symbol_get_const_boolean ($2); break; default: - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); break; } } @@ -382,7 +472,7 @@ unary_expression { $$ = $3; if ($$->const_int_set) { - $$->base_type = gi_source_basic_type_new ("gint64"); + $$->base_type = gi_source_basic_type_new ($$->const_int_is_unsigned ? "guint64" : "gint64"); } } | INTUL_CONST '(' unary_expression ')' @@ -394,12 +484,12 @@ unary_expression } | SIZEOF unary_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | SIZEOF '(' type_name ')' { ctype_free ($3); - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } ; @@ -447,13 +537,13 @@ multiplicative_expression : cast_expression | multiplicative_expression '*' cast_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int * $3->const_int; } | multiplicative_expression '/' cast_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; if ($3->const_int != 0) { $$->const_int = $1->const_int / $3->const_int; @@ -461,7 +551,7 @@ multiplicative_expression } | multiplicative_expression '%' cast_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; if ($3->const_int != 0) { $$->const_int = $1->const_int % $3->const_int; @@ -473,13 +563,13 @@ additive_expression : multiplicative_expression | additive_expression '+' multiplicative_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int + $3->const_int; } | additive_expression '-' multiplicative_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int - $3->const_int; } @@ -489,7 +579,7 @@ shift_expression : additive_expression | shift_expression SL additive_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int << $3->const_int; @@ -501,7 +591,7 @@ shift_expression } | shift_expression SR additive_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int >> $3->const_int; } @@ -511,25 +601,25 @@ relational_expression : shift_expression | relational_expression '<' shift_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int < $3->const_int; } | relational_expression '>' shift_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int > $3->const_int; } | relational_expression LTEQ shift_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int <= $3->const_int; } | relational_expression GTEQ shift_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int >= $3->const_int; } @@ -539,13 +629,13 @@ equality_expression : relational_expression | equality_expression EQ relational_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int == $3->const_int; } | equality_expression NOTEQ relational_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int != $3->const_int; } @@ -555,7 +645,7 @@ and_expression : equality_expression | and_expression '&' equality_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int & $3->const_int; } @@ -565,7 +655,7 @@ exclusive_or_expression : and_expression | exclusive_or_expression '^' and_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int ^ $3->const_int; } @@ -575,7 +665,7 @@ inclusive_or_expression : exclusive_or_expression | inclusive_or_expression '|' exclusive_or_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = $1->const_int | $3->const_int; } @@ -585,7 +675,7 @@ logical_and_expression : inclusive_or_expression | logical_and_expression ANDAND inclusive_or_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = gi_source_symbol_get_const_boolean ($1) && @@ -597,7 +687,7 @@ logical_or_expression : logical_and_expression | logical_or_expression OROR logical_and_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_CONST, scanner->current_file, lineno); $$->const_int_set = TRUE; $$->const_int = gi_source_symbol_get_const_boolean ($1) || @@ -617,7 +707,7 @@ assignment_expression : conditional_expression | unary_expression assignment_operator assignment_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } ; @@ -640,7 +730,7 @@ expression | expression ',' assignment_expression | EXTENSION expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } ; @@ -814,11 +904,12 @@ type_specifier struct_or_union_specifier : struct_or_union identifier_or_typedef_name '{' struct_declaration_list '}' { + GISourceSymbol *sym; $$ = $1; $$->name = $2; $$->child_list = $4; - GISourceSymbol *sym = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + sym = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); if ($$->type == CTYPE_STRUCT) { sym->type = CSYMBOL_TYPE_STRUCT; } else if ($$->type == CTYPE_UNION) { @@ -917,12 +1008,12 @@ struct_declarator_list struct_declarator : /* empty, support for anonymous structs and unions */ { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | declarator | ':' constant_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); } | declarator ':' constant_expression { @@ -998,7 +1089,7 @@ enumerator_list enumerator : identifier { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_OBJECT, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_OBJECT, scanner->current_file, lineno); $$->ident = $1; $$->const_int_set = TRUE; $$->const_int = ++last_enum_value; @@ -1006,7 +1097,7 @@ enumerator } | identifier '=' constant_expression { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_OBJECT, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_OBJECT, scanner->current_file, lineno); $$->ident = $1; $$->const_int_set = TRUE; $$->const_int = $3->const_int; @@ -1053,7 +1144,7 @@ declarator direct_declarator : identifier { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); $$->ident = $1; } | '(' declarator ')' @@ -1160,25 +1251,25 @@ parameter_declaration } | declaration_specifiers { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); $$->base_type = $1; } | ELLIPSIS { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_ELLIPSIS, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_ELLIPSIS, scanner->current_file, lineno); } ; identifier_list : identifier { - GISourceSymbol *sym = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + GISourceSymbol *sym = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); sym->ident = $1; $$ = g_list_append (NULL, sym); } | identifier_list ',' identifier { - GISourceSymbol *sym = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + GISourceSymbol *sym = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); sym->ident = $3; $$ = g_list_append ($1, sym); } @@ -1192,7 +1283,7 @@ type_name abstract_declarator : pointer { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); gi_source_symbol_merge_type ($$, $1); } | direct_abstract_declarator @@ -1210,12 +1301,12 @@ direct_abstract_declarator } | '[' ']' { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); gi_source_symbol_merge_type ($$, gi_source_array_new (NULL)); } | '[' assignment_expression ']' { - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); gi_source_symbol_merge_type ($$, gi_source_array_new ($2)); } | direct_abstract_declarator '[' ']' @@ -1231,7 +1322,7 @@ direct_abstract_declarator | '(' ')' { GISourceType *func = gi_source_function_new (); - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); gi_source_symbol_merge_type ($$, func); } | '(' parameter_list ')' @@ -1241,7 +1332,7 @@ direct_abstract_declarator if ($2 != NULL && ($2->next != NULL || ((GISourceSymbol *) $2->data)->base_type->type != CTYPE_VOID)) { func->child_list = $2; } - $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_filename, lineno); + $$ = gi_source_symbol_new (CSYMBOL_TYPE_INVALID, scanner->current_file, lineno); gi_source_symbol_merge_type ($$, func); } | direct_abstract_declarator '(' ')' @@ -1390,7 +1481,7 @@ function_macro_define object_macro_define : object_macro constant_expression { - if ($2->const_int_set || $2->const_double_set || $2->const_string != NULL) { + if ($2->const_int_set || $2->const_boolean_set || $2->const_double_set || $2->const_string != NULL) { $2->ident = $1; gi_source_scanner_add_symbol (scanner, $2); gi_source_symbol_unref ($2); @@ -1398,9 +1489,55 @@ object_macro_define } ; +preproc_conditional + : IFDEF_GI_SCANNER + { + push_conditional (scanner, FOR_GI_SCANNER); + update_skipping (scanner); + } + | IFNDEF_GI_SCANNER + { + push_conditional (scanner, NOT_GI_SCANNER); + update_skipping (scanner); + } + | IFDEF_COND + { + warn_if_cond_has_gi_scanner (scanner, yytext); + push_conditional (scanner, IRRELEVANT); + } + | IFNDEF_COND + { + warn_if_cond_has_gi_scanner (scanner, yytext); + push_conditional (scanner, IRRELEVANT); + } + | IF_COND + { + warn_if_cond_has_gi_scanner (scanner, yytext); + push_conditional (scanner, IRRELEVANT); + } + | ELIF_COND + { + warn_if_cond_has_gi_scanner (scanner, yytext); + pop_conditional (scanner); + push_conditional (scanner, IRRELEVANT); + update_skipping (scanner); + } + | ELSE_COND + { + toggle_conditional (scanner); + update_skipping (scanner); + } + | ENDIF_COND + { + pop_conditional (scanner); + update_skipping (scanner); + } + ; + macro : function_macro_define | object_macro_define + | preproc_conditional | error ; @@ -1413,7 +1550,7 @@ yyerror (GISourceScanner *scanner, const char *s) if (!scanner->macro_scan) { fprintf(stderr, "%s:%d: %s in '%s' at '%s'\n", - scanner->current_filename, lineno, s, linebuf, yytext); + g_file_get_parse_name (scanner->current_file), lineno, s, linebuf, yytext); } } @@ -1430,14 +1567,19 @@ eat_hspace (FILE * f) } static int -eat_line (FILE * f, int c) +pass_line (FILE * f, int c, + FILE *out) { while (c != EOF && c != '\n') { + if (out) + fputc (c, out); c = fgetc (f); } if (c == '\n') { + if (out) + fputc (c, out); c = fgetc (f); if (c == ' ' || c == '\t') { @@ -1448,6 +1590,12 @@ eat_line (FILE * f, int c) } static int +eat_line (FILE * f, int c) +{ + return pass_line (f, c, NULL); +} + +static int read_identifier (FILE * f, int c, char **identifier) { GString *id = g_string_new (""); @@ -1468,9 +1616,9 @@ gi_source_scanner_parse_macros (GISourceScanner *scanner, GList *filenames) FILE *fmacros = fdopen (g_file_open_tmp ("gen-introspect-XXXXXX.h", &tmp_name, &error), "w+"); + GList *l; g_unlink (tmp_name); - GList *l; for (l = filenames; l != NULL; l = l->next) { FILE *f = fopen (l->data, "r"); @@ -1479,6 +1627,7 @@ gi_source_scanner_parse_macros (GISourceScanner *scanner, GList *filenames) GString *define_line; char *str; gboolean error_line = FALSE; + gboolean end_of_word; int c = eat_hspace (f); while (c != EOF) { @@ -1497,7 +1646,22 @@ gi_source_scanner_parse_macros (GISourceScanner *scanner, GList *filenames) c = eat_hspace (f); c = read_identifier (f, c, &str); - if (strcmp (str, "define") != 0 || (c != ' ' && c != '\t')) + end_of_word = (c == ' ' || c == '\t' || c == '\n' || c == EOF); + if (end_of_word && + (g_str_equal (str, "if") || + g_str_equal (str, "endif") || + g_str_equal (str, "ifndef") || + g_str_equal (str, "ifdef") || + g_str_equal (str, "else") || + g_str_equal (str, "elif"))) + { + fprintf (fmacros, "#%s ", str); + g_free (str); + c = pass_line (f, c, fmacros); + line++; + continue; + } + else if (strcmp (str, "define") != 0 || !end_of_word) { g_free (str); /* ignore line */ diff --git a/giscanner/sectionparser.py b/giscanner/sectionparser.py new file mode 100644 index 00000000..ffe41afc --- /dev/null +++ b/giscanner/sectionparser.py @@ -0,0 +1,152 @@ +# -*- Mode: Python -*- +# Copyright (C) 2013 Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# + +import re +from . import ast +from .utils import to_underscores + + +class SectionsFile(object): + def __init__(self, sections): + self.sections = sections + + +class Section(object): + def __init__(self): + self.file = None + self.title = None + self.includes = None + self.subsections = [] + + +class Subsection(object): + def __init__(self, name): + self.name = name + self.symbols = [] + + +def parse_sections_file(lines): + sections = [] + current_section = None + current_subsection = None + + for line in lines: + line = line.rstrip() + + if not line or line.isspace(): + continue + + if line == "<SECTION>": + current_section = Section() + sections.append(current_section) + current_subsection = Subsection(None) + current_section.subsections.append(current_subsection) + continue + + if line == "</SECTION>": + current_section = None + continue + + match = re.match(r"<FILE>(?P<contents>.*)</FILE>", line) + if match: + current_section.file = match.groupdict['contents'] + continue + + match = re.match(r"<TITLE>(?P<contents>.*)</TITLE>", line) + if match: + current_section.title = match.groupdict['contents'] + continue + + match = re.match(r"<INCLUDE>(?P<contents>.*)</INCLUDE>", line) + if match: + current_section.includes = match.groupdict['contents'] + continue + + match = re.match(r"<SUBSECTION(?: (?P<name>.*))?>", line) + if match: + current_subsection = Subsection(match.groupdict.get('name', None)) + current_section.subsections.append(current_subsection) + continue + + if line.startswith("<") and line.endswith(">"): + # Other directive to gtk-doc, not a symbol. + continue + + current_subsection.symbols.append(line) + + return SectionsFile(sections) + + +def write_sections_file(f, sections_file): + for section in sections_file.sections: + f.write("\n<SECTION>\n") + if section.file is not None: + f.write("<FILE>%s</FILE>\n" % (section.file, )) + if section.title is not None: + f.write("<TITLE>%s</TITLE>\n" % (section.title, )) + if section.includes is not None: + f.write("<INCLUDE>%s</INCLUDE>\n" % (section.includes, )) + + is_first_subsection = True + for subsection in section.subsections: + if subsection.name is not None: + f.write("<SUBSECTION %s>\n" % (subsection.name, )) + elif not is_first_subsection: + f.write("\n<SUBSECTION>\n") + + is_first_subsection = False + + for symbol in subsection.symbols: + f.write(symbol + "\n") + + +def generate_sections_file(transformer): + ns = transformer.namespace + + sections = [] + + def new_section(file_, title): + section = Section() + section.file = file_ + section.title = title + section.subsections.append(Subsection(None)) + sections.append(section) + return section + + def append_symbol(section, sym): + section.subsections[0].symbols.append(sym) + + general_section = new_section("main", "Main") + + for node in ns.itervalues(): + if isinstance(node, ast.Function): + append_symbol(general_section, node.symbol) + elif isinstance(node, (ast.Class, ast.Interface)): + gtype_name = node.gtype_name + file_name = to_underscores(gtype_name).replace('_', '-').lower() + section = new_section(file_name, gtype_name) + append_symbol(section, gtype_name) + append_symbol(section, node.glib_type_struct.target_giname.replace('.', '')) + + for meth in node.methods: + append_symbol(section, meth.symbol) + for meth in node.static_methods: + append_symbol(section, meth.symbol) + + return SectionsFile(sections) diff --git a/giscanner/shlibs.py b/giscanner/shlibs.py index 9579e7e6..21fcafd0 100644 --- a/giscanner/shlibs.py +++ b/giscanner/shlibs.py @@ -20,13 +20,13 @@ # import os -import re import platform +import re import subprocess -import os from .utils import get_libtool_command, extract_libtool_shlib + # For .la files, the situation is easy. def _resolve_libtool(options, binary, libraries): shlibs = [] @@ -37,6 +37,7 @@ def _resolve_libtool(options, binary, libraries): return shlibs + # Assume ldd output is something vaguely like # # libpangoft2-1.0.so.0 => /usr/lib/libpangoft2-1.0.so.0 (0x006c1000) @@ -48,9 +49,14 @@ def _resolve_libtool(options, binary, libraries): # The negative lookbehind at the start is to avoid problems if someone # is crazy enough to name a library liblib<foo> when lib<foo> exists. # +# Match absolute paths on OS X to conform to how libraries are usually +# referenced on OS X systems. def _ldd_library_pattern(library_name): - return re.compile("(?<![A-Za-z0-9_-])(lib*%s[^A-Za-z0-9_-][^\s\(\)]*)" - % re.escape(library_name)) + pattern = "(?<![A-Za-z0-9_-])(lib*%s[^A-Za-z0-9_-][^\s\(\)]*)" + if platform.system() == 'Darwin': + pattern = "([^\s]*lib*%s[^A-Za-z0-9_-][^\s\(\)]*)" + return re.compile(pattern % re.escape(library_name)) + # This is a what we do for non-la files. We assume that we are on an # ELF-like system where ldd exists and the soname extracted with ldd is @@ -69,7 +75,7 @@ def _resolve_non_libtool(options, binary, libraries): if not libraries: return [] - if os.uname()[0] == 'OpenBSD': + if platform.platform().startswith('OpenBSD'): # Hack for OpenBSD when using the ports' libtool which uses slightly # different directories to store the libraries in. So rewite binary.args[0] # by inserting '.libs/'. @@ -119,6 +125,7 @@ def _resolve_non_libtool(options, binary, libraries): return shlibs + # We want to resolve a set of library names (the <foo> of -l<foo>) # against a library to find the shared library name. The shared # library name is suppose to be what you pass to dlopen() (or diff --git a/giscanner/sourcescanner.c b/giscanner/sourcescanner.c index 1b775b46..7de891f5 100644 --- a/giscanner/sourcescanner.c +++ b/giscanner/sourcescanner.c @@ -25,11 +25,11 @@ #include <gio/gio.h> GISourceSymbol * -gi_source_symbol_new (GISourceSymbolType type, const gchar *filename, int line) +gi_source_symbol_new (GISourceSymbolType type, GFile *file, int line) { GISourceSymbol *s = g_slice_new0 (GISourceSymbol); s->ref_count = 1; - s->source_filename = g_strdup (filename); + s->source_filename = g_file_get_parse_name (file); s->type = type; s->line = line; return s; @@ -45,6 +45,35 @@ ctype_free (GISourceType * type) } GISourceSymbol * +gi_source_symbol_copy (GISourceSymbol * symbol) +{ + GFile *source_file = g_file_new_for_path (symbol->source_filename); + GISourceSymbol *new_symbol = gi_source_symbol_new (symbol->type, + source_file, + symbol->line); + new_symbol->ident = g_strdup (symbol->ident); + + if (symbol->base_type) + new_symbol->base_type = gi_source_type_copy (symbol->base_type); + + if (symbol->const_int_set) { + new_symbol->const_int = symbol->const_int; + new_symbol->const_int_is_unsigned = symbol->const_int_is_unsigned; + new_symbol->const_int_set = TRUE; + } else if (symbol->const_boolean_set) { + new_symbol->const_boolean = symbol->const_boolean; + new_symbol->const_boolean_set = TRUE; + } else if (symbol->const_double_set) { + new_symbol->const_double = symbol->const_double; + new_symbol->const_double_set = TRUE; + } else if (symbol->const_string != NULL) { + new_symbol->const_string = g_strdup (symbol->const_string); + } + + return new_symbol; +} + +GISourceSymbol * gi_source_symbol_ref (GISourceSymbol * symbol) { symbol->ref_count++; @@ -188,11 +217,10 @@ gi_source_scanner_new (void) scanner = g_slice_new0 (GISourceScanner); scanner->typedef_table = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, NULL); - scanner->struct_or_union_or_enum_table = - g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, (GDestroyNotify)gi_source_symbol_unref); - + g_free, NULL); + scanner->files = g_hash_table_new_full (g_file_hash, (GEqualFunc)g_file_equal, + g_object_unref, NULL); + g_queue_init (&scanner->conditionals); return scanner; } @@ -207,19 +235,18 @@ gi_source_comment_free (GISourceComment *comment) void gi_source_scanner_free (GISourceScanner *scanner) { - g_free (scanner->current_filename); + g_object_unref (scanner->current_file); g_hash_table_destroy (scanner->typedef_table); - g_hash_table_destroy (scanner->struct_or_union_or_enum_table); g_slist_foreach (scanner->comments, (GFunc)gi_source_comment_free, NULL); g_slist_free (scanner->comments); g_slist_foreach (scanner->symbols, (GFunc)gi_source_symbol_unref, NULL); g_slist_free (scanner->symbols); - g_list_foreach (scanner->filenames, (GFunc)g_free, NULL); - g_list_free (scanner->filenames); + g_hash_table_unref (scanner->files); + g_queue_clear (&scanner->conditionals); } gboolean @@ -241,29 +268,18 @@ void gi_source_scanner_add_symbol (GISourceScanner *scanner, GISourceSymbol *symbol) { - gboolean found_filename = FALSE; - GList *l; - GFile *current_file; - - g_assert (scanner->current_filename); - current_file = g_file_new_for_path (scanner->current_filename); - - for (l = scanner->filenames; l != NULL; l = l->next) + if (scanner->skipping) { - GFile *file = g_file_new_for_path (l->data); - - if (g_file_equal (file, current_file)) - { - found_filename = TRUE; - g_object_unref (file); - break; - } - g_object_unref (file); + g_debug ("skipping symbol due to __GI_SCANNER__ cond: %s", symbol->ident); + return; } - if (found_filename || scanner->macro_scan) + g_assert (scanner->current_file); + + if (scanner->macro_scan || g_hash_table_contains (scanner->files, scanner->current_file)) scanner->symbols = g_slist_prepend (scanner->symbols, - gi_source_symbol_ref (symbol)); + gi_source_symbol_ref (symbol)); + g_assert (symbol->source_filename != NULL); switch (symbol->type) @@ -273,28 +289,48 @@ gi_source_scanner_add_symbol (GISourceScanner *scanner, g_strdup (symbol->ident), GINT_TO_POINTER (TRUE)); break; - case CSYMBOL_TYPE_STRUCT: - case CSYMBOL_TYPE_UNION: - case CSYMBOL_TYPE_ENUM: - g_hash_table_insert (scanner->struct_or_union_or_enum_table, - g_strdup (symbol->ident), - gi_source_symbol_ref (symbol)); - break; default: break; } +} - g_object_unref (current_file); +void +gi_source_scanner_take_comment (GISourceScanner *scanner, + GISourceComment *comment) +{ + if (scanner->skipping) + { + g_debug ("skipping comment due to __GI_SCANNER__ cond"); + gi_source_comment_free (comment); + return; + } + + scanner->comments = g_slist_prepend (scanner->comments, + comment); } +/** + * gi_source_scanner_get_symbols: + * @scanner: scanner instance + * + * Returns: (transfer container): List of GISourceSymbol. + * Free resulting list with g_slist_free(). + */ GSList * gi_source_scanner_get_symbols (GISourceScanner *scanner) { - return g_slist_reverse (scanner->symbols); + return g_slist_reverse (g_slist_copy (scanner->symbols)); } +/** + * gi_source_scanner_get_comments: + * @scanner: scanner instance + * + * Returns: (transfer container): List of GISourceComment. + * Free resulting list with g_slist_free(). + */ GSList * gi_source_scanner_get_comments(GISourceScanner *scanner) { - return g_slist_reverse (scanner->comments); + return g_slist_reverse (g_slist_copy (scanner->comments)); } diff --git a/giscanner/sourcescanner.h b/giscanner/sourcescanner.h index df16cf6a..9e371312 100644 --- a/giscanner/sourcescanner.h +++ b/giscanner/sourcescanner.h @@ -25,6 +25,7 @@ #include <glib.h> #include <stdio.h> +#include <gio/gio.h> G_BEGIN_DECLS @@ -105,30 +106,33 @@ struct _GISourceComment struct _GISourceScanner { - char *current_filename; + GFile *current_file; gboolean macro_scan; gboolean private; /* set by gtk-doc comment <private>/<public> */ gboolean flags; /* set by gtk-doc comment <flags> */ GSList *symbols; - GList *filenames; + GHashTable *files; GSList *comments; /* _GIComment */ GHashTable *typedef_table; - GHashTable *struct_or_union_or_enum_table; + gboolean skipping; + GQueue conditionals; }; struct _GISourceSymbol { int ref_count; GISourceSymbolType type; - int id; char *ident; GISourceType *base_type; gboolean const_int_set; gboolean private; gint64 const_int; /* 64-bit we can handle signed and unsigned 32-bit values */ + gboolean const_int_is_unsigned; char *const_string; gboolean const_double_set; double const_double; + gboolean const_boolean_set; + int const_boolean; char *source_filename; int line; }; @@ -158,14 +162,17 @@ GSList * gi_source_scanner_get_symbols (GISourceScanner *scanne GSList * gi_source_scanner_get_comments (GISourceScanner *scanner); void gi_source_scanner_free (GISourceScanner *scanner); -GISourceSymbol * gi_source_symbol_new (GISourceSymbolType type, const gchar *filename, int line); +GISourceSymbol * gi_source_symbol_new (GISourceSymbolType type, GFile *file, int line); gboolean gi_source_symbol_get_const_boolean (GISourceSymbol *symbol); GISourceSymbol * gi_source_symbol_ref (GISourceSymbol *symbol); void gi_source_symbol_unref (GISourceSymbol *symbol); +GISourceSymbol * gi_source_symbol_copy (GISourceSymbol *symbol); /* Private */ void gi_source_scanner_add_symbol (GISourceScanner *scanner, GISourceSymbol *symbol); +void gi_source_scanner_take_comment (GISourceScanner *scanner, + GISourceComment *comment); gboolean gi_source_scanner_is_typedef (GISourceScanner *scanner, const char *name); void gi_source_symbol_merge_type (GISourceSymbol *symbol, diff --git a/giscanner/sourcescanner.py b/giscanner/sourcescanner.py index db282f84..dab16027 100644 --- a/giscanner/sourcescanner.py +++ b/giscanner/sourcescanner.py @@ -32,6 +32,10 @@ with LibtoolImporter(None, None): else: from giscanner._giscanner import SourceScanner as CSourceScanner +HEADER_EXTS = ['.h', '.hpp', '.hxx'] +SOURCE_EXTS = ['.c', '.cpp', '.cc', '.cxx'] +ALL_EXTS = SOURCE_EXTS + HEADER_EXTS + (CSYMBOL_TYPE_INVALID, CSYMBOL_TYPE_ELLIPSIS, CSYMBOL_TYPE_CONST, @@ -89,8 +93,7 @@ def symbol_type_name(symbol_type): CSYMBOL_TYPE_UNION: 'union', CSYMBOL_TYPE_ENUM: 'enum', CSYMBOL_TYPE_TYPEDEF: 'typedef', - CSYMBOL_TYPE_MEMBER: 'member', - }.get(symbol_type) + CSYMBOL_TYPE_MEMBER: 'member'}.get(symbol_type) def ctype_name(ctype): @@ -104,8 +107,7 @@ def ctype_name(ctype): CTYPE_ENUM: 'enum', CTYPE_POINTER: 'pointer', CTYPE_ARRAY: 'array', - CTYPE_FUNCTION: 'function', - }.get(ctype) + CTYPE_FUNCTION: 'function'}.get(ctype) class SourceType(object): @@ -152,8 +154,8 @@ class SourceType(object): class SourceSymbol(object): - __members__ = ['const_int', 'const_double', 'const_string', 'ident', - 'type', 'base_type'] + __members__ = ['const_int', 'const_double', 'const_string', 'const_boolean', + 'ident', 'type', 'base_type'] def __init__(self, scanner, symbol): self._scanner = scanner @@ -184,6 +186,10 @@ class SourceSymbol(object): return self._symbol.const_string @property + def const_boolean(self): + return self._symbol.const_boolean + + @property def ident(self): return self._symbol.ident @@ -223,8 +229,9 @@ class SourceScanner(object): # Public API - def set_cpp_options(self, includes, defines, undefines): - for prefix, args in [('-I', includes), + def set_cpp_options(self, includes, defines, undefines, cflags=[]): + self._cpp_options.extend(cflags) + for prefix, args in [('-I', [os.path.realpath(f) for f in includes]), ('-D', defines), ('-U', undefines)]: for arg in (args or []): @@ -234,15 +241,14 @@ class SourceScanner(object): def parse_files(self, filenames): for filename in filenames: - filename = os.path.abspath(filename) + # self._scanner expects file names to be canonicalized and symlinks to be resolved + filename = os.path.realpath(filename) self._scanner.append_filename(filename) self._filenames.append(filename) headers = [] - for filename in filenames: - if (filename.endswith('.c') or filename.endswith('.cpp') or - filename.endswith('.cc') or filename.endswith('.cxx')): - filename = os.path.abspath(filename) + for filename in self._filenames: + if os.path.splitext(filename)[1] in SOURCE_EXTS: self._scanner.lex_filename(filename) else: headers.append(filename) @@ -251,7 +257,8 @@ class SourceScanner(object): def parse_macros(self, filenames): self._scanner.set_macro_scan(True) - self._scanner.parse_macros(filenames) + # self._scanner expects file names to be canonicalized and symlinks to be resolved + self._scanner.parse_macros([os.path.realpath(f) for f in filenames]) self._scanner.set_macro_scan(False) def get_symbols(self): @@ -262,7 +269,7 @@ class SourceScanner(object): return self._scanner.get_comments() def dump(self): - print '-'*30 + print '-' * 30 for symbol in self._scanner.get_symbols(): print symbol.ident, symbol.base_type.name, symbol.type @@ -274,10 +281,19 @@ class SourceScanner(object): defines = ['__GI_SCANNER__'] undefs = [] - cpp_args = os.environ.get('CC', 'cc').split() + cpp_args = os.environ.get('CC', 'cc').split() # support CC="ccache gcc" + if 'cl' in cpp_args: + # The Microsoft compiler/preprocessor (cl) does not accept + # source input from stdin (the '-' flag), so we need + # some help from gcc from MinGW/Cygwin or so. + # Note that the generated dumper program is + # still built and linked by Visual C++. + cpp_args = ['gcc'] + cpp_args += os.environ.get('CPPFLAGS', '').split() + cpp_args += os.environ.get('CFLAGS', '').split() cpp_args += ['-E', '-C', '-I.', '-'] - cpp_args += self._cpp_options + proc = subprocess.Popen(cpp_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) @@ -289,12 +305,11 @@ class SourceScanner(object): for undef in undefs: proc.stdin.write('#undef %s\n' % (undef, )) for filename in filenames: - filename = os.path.abspath(filename) proc.stdin.write('#include <%s>\n' % (filename, )) proc.stdin.close() - tmp = tempfile.mktemp() - fp = open(tmp, 'w+') + tmp_fd, tmp_name = tempfile.mkstemp() + fp = os.fdopen(tmp_fd, 'w+b') while True: data = proc.stdout.read(4096) if data is None: @@ -311,4 +326,4 @@ class SourceScanner(object): self._scanner.parse_file(fp.fileno()) fp.close() - os.unlink(tmp) + os.unlink(tmp_name) diff --git a/giscanner/testcodegen.py b/giscanner/testcodegen.py index f304dc7a..1ed247c7 100644 --- a/giscanner/testcodegen.py +++ b/giscanner/testcodegen.py @@ -27,12 +27,14 @@ DEFAULT_C_VALUES = {ast.TYPE_ANY: 'NULL', ast.TYPE_FILENAME: '""', ast.TYPE_GTYPE: 'g_object_get_type ()'} + def get_default_for_typeval(typeval): default = DEFAULT_C_VALUES.get(typeval) if default: return default return "0" + def uscore_from_type(typeval): if typeval.target_fundamental: return typeval.target_fundamental.replace(' ', '_') @@ -41,6 +43,7 @@ def uscore_from_type(typeval): else: assert False, typeval + class EverythingCodeGenerator(object): def __init__(self, out_h_filename, out_c_filename): diff --git a/giscanner/transformer.py b/giscanner/transformer.py index 6afad889..80265dd8 100644 --- a/giscanner/transformer.py +++ b/giscanner/transformer.py @@ -34,6 +34,7 @@ from .sourcescanner import ( CSYMBOL_TYPE_MEMBER, CSYMBOL_TYPE_ELLIPSIS, CSYMBOL_TYPE_CONST, TYPE_QUALIFIER_CONST, TYPE_QUALIFIER_VOLATILE) + class TransformerException(Exception): pass @@ -54,14 +55,14 @@ class Transformer(object): self._namespace = namespace self._pkg_config_packages = set() self._typedefs_ns = {} - self._includes = {} # <string namespace -> Namespace> - self._include_names = set() # string namespace + self._parsed_includes = {} # <string namespace -> Namespace> self._includepaths = [] self._passthrough_mode = False - self._annotations = {} - def get_includes(self): - return self._include_names + # Cache a list of struct/unions in C's "tag namespace". This helps + # manage various orderings of typedefs and structs. See: + # https://bugzilla.gnome.org/show_bug.cgi?id=581525 + self._tag_ns = {} def get_pkgconfig_packages(self): return self._pkg_config_packages @@ -72,9 +73,6 @@ class Transformer(object): def set_passthrough_mode(self): self._passthrough_mode = True - def set_annotations(self, annotations): - self._annotations = annotations - def _append_new_node(self, node): original = self._namespace.get(node.name) # Special case constants here; we allow duplication to sort-of @@ -83,6 +81,12 @@ class Transformer(object): # modules will just depend on that. if isinstance(original, ast.Constant) and isinstance(node, ast.Constant): pass + elif original is node: + # Ignore attempts to add the same node to the namespace. This can + # happen when parsing typedefs and structs in particular orderings: + # typedef struct _Foo Foo; + # struct _Foo {...}; + pass elif original: positions = set() positions.update(original.file_positions) @@ -98,50 +102,52 @@ class Transformer(object): # https://bugzilla.gnome.org/show_bug.cgi?id=550616 if symbol.ident in ['gst_g_error_get_type']: continue - node = self._traverse_one(symbol) - if node: - self._append_new_node(node) - # Now look through the namespace for things like - # typedef struct _Foo Foo; - # where we've never seen the struct _Foo. Just create - # an empty structure for these as "disguised" - # If we do have a class/interface, merge fields - for typedef, compound in self._typedefs_ns.iteritems(): - ns_compound = self._namespace.get(compound.name) - if not ns_compound: - ns_compound = self._namespace.get('_' + compound.name) - if (not ns_compound and isinstance(compound, (ast.Record, ast.Union)) - and len(compound.fields) == 0): - disguised = ast.Record(compound.name, typedef, disguised=True) - self._namespace.append(disguised) - elif not ns_compound: - self._namespace.append(compound) - elif isinstance(ns_compound, (ast.Record, ast.Union)) and len(ns_compound.fields) == 0: - ns_compound.fields = compound.fields - self._typedefs_ns = None + try: + node = self._traverse_one(symbol) + except TransformerException as e: + message.warn_symbol(symbol, e) + continue + + if node and node.name: + self._append_new_node(node) + if isinstance(node, ast.Compound) and node.tag_name and \ + node.tag_name not in self._tag_ns: + self._tag_ns[node.tag_name] = node + + # Run through the tag namespace looking for structs that have not been + # promoted into the main namespace. In this case we simply promote them + # with their struct tag. + for tag_name, struct in self._tag_ns.iteritems(): + if not struct.name: + try: + name = self.strip_identifier(tag_name) + struct.name = name + self._append_new_node(struct) + except TransformerException as e: + message.warn_node(node, e) def set_include_paths(self, paths): self._includepaths = list(paths) def register_include(self, include): - if include in self._include_names: + if include in self._namespace.includes: return + self._namespace.includes.add(include) filename = self._find_include(include) self._parse_include(filename) - self._include_names.add(include) def register_include_uninstalled(self, include_path): basename = os.path.basename(include_path) if not basename.endswith('.gir'): - raise SystemExit( -"Include path %r must be a filename path ending in .gir" % (include_path, )) + raise SystemExit("Include path %r must be a filename path " + "ending in .gir" % (include_path, )) girname = basename[:-4] include = ast.Include.from_string(girname) - if include in self._include_names: + if include in self._namespace.includes: return + self._namespace.includes.add(include) self._parse_include(include_path, uninstalled=True) - self._include_names.add(include) def lookup_giname(self, name): """Given a name of the form Foo or Bar.Foo, @@ -151,11 +157,16 @@ namespaces.""" if '.' not in name: return self._namespace.get(name) else: - (ns, name) = name.split('.', 1) + (ns, giname) = name.split('.', 1) if ns == self._namespace.name: - return self._namespace.get(name) - include = self._includes[ns] - return include.get(name) + return self._namespace.get(giname) + # Fallback to the main namespace if not a dependency and matches a prefix + if ns in self._namespace.identifier_prefixes and not ns in self._parsed_includes: + message.warn(("Deprecated reference to identifier " + + "prefix %s in GIName %s") % (ns, name)) + return self._namespace.get(giname) + include = self._parsed_includes[ns] + return include.get(giname) def lookup_typenode(self, typeobj): """Given a Type object, if it points to a giname, @@ -178,8 +189,7 @@ None.""" path = os.path.join(d, girname) if os.path.exists(path): return path - sys.stderr.write("Couldn't find include %r (search path: %r)\n"\ - % (girname, searchdirs)) + sys.stderr.write("Couldn't find include %r (search path: %r)\n" % (girname, searchdirs)) sys.exit(1) @classmethod @@ -191,7 +201,7 @@ None.""" self._parse_include(filename) parser = self._cachestore.load(filename) self._namespace = parser.get_namespace() - del self._includes[self._namespace.name] + del self._parsed_includes[self._namespace.name] return self def _parse_include(self, filename, uninstalled=False): @@ -204,20 +214,22 @@ None.""" if self._cachestore is not None: self._cachestore.store(filename, parser) - for include in parser.get_includes(): - self.register_include(include) + for include in parser.get_namespace().includes: + if include.name not in self._parsed_includes: + dep_filename = self._find_include(include) + self._parse_include(dep_filename) if not uninstalled: - for pkg in parser.get_pkgconfig_packages(): + for pkg in parser.get_namespace().exported_packages: self._pkg_config_packages.add(pkg) namespace = parser.get_namespace() - self._includes[namespace.name] = namespace + self._parsed_includes[namespace.name] = namespace def _iter_namespaces(self): """Return an iterator over all included namespaces; the currently-scanned namespace is first.""" yield self._namespace - for ns in self._includes.itervalues(): + for ns in self._parsed_includes.itervalues(): yield ns def _sort_matches(self, x, y): @@ -229,7 +241,7 @@ currently-scanned namespace is first.""" def _split_c_string_for_namespace_matches(self, name, is_identifier=False): matches = [] # Namespaces which might contain this name - unprefixed_namespaces = [] # Namespaces with no prefix, last resort + unprefixed_namespaces = [] # Namespaces with no prefix, last resort for ns in self._iter_namespaces(): if is_identifier: prefixes = ns.identifier_prefixes @@ -286,7 +298,7 @@ raise ValueError.""" ident = ident[1:] try: matches = self.split_ctype_namespaces(ident) - except ValueError, e: + except ValueError as e: raise TransformerException(str(e)) for ns, name in matches: if ns is self._namespace: @@ -295,8 +307,7 @@ raise ValueError.""" return name (ns, name) = matches[-1] raise TransformerException( - "Skipping foreign identifier %r from namespace %s" % ( - ident, ns.name, )) + "Skipping foreign identifier %r from namespace %s" % (ident, ns.name, )) return None def _strip_symbol(self, symbol): @@ -306,7 +317,7 @@ raise ValueError.""" ident = ident[1:] try: (ns, name) = self.split_csymbol(ident) - except ValueError, e: + except ValueError as e: raise TransformerException(str(e)) if ns != self._namespace: raise TransformerException( @@ -325,18 +336,15 @@ raise ValueError.""" elif stype == CSYMBOL_TYPE_TYPEDEF: return self._create_typedef(symbol) elif stype == CSYMBOL_TYPE_STRUCT: - return self._create_struct(symbol) + return self._create_tag_ns_compound(ast.Record, symbol) elif stype == CSYMBOL_TYPE_ENUM: return self._create_enum(symbol) elif stype == CSYMBOL_TYPE_MEMBER: return self._create_member(symbol, parent_symbol) elif stype == CSYMBOL_TYPE_UNION: - return self._create_union(symbol) + return self._create_tag_ns_compound(ast.Union, symbol) elif stype == CSYMBOL_TYPE_CONST: - # Don't parse constants which are marked (skip) - docblock = self._annotations.get(symbol.ident) - if not docblock or not 'skip' in docblock.options: - return self._create_const(symbol) + return self._create_const(symbol) # Ignore variable declarations in the header elif stype == CSYMBOL_TYPE_OBJECT: pass @@ -381,21 +389,13 @@ raise ValueError.""" # Ok, the enum members don't have a consistent prefix # among them, so let's just remove the global namespace # prefix. - try: - name = self._strip_symbol(child) - except TransformerException, e: - message.warn_symbol(symbol, e) - return None + name = self._strip_symbol(child) members.append(ast.Member(name.lower(), child.const_int, child.ident, None)) - try: - enum_name = self.strip_identifier(symbol.ident) - except TransformerException, e: - message.warn_symbol(symbol, e) - return None + enum_name = self.strip_identifier(symbol.ident) if symbol.base_type.is_bitfield: klass = ast.Bitfield else: @@ -408,13 +408,9 @@ raise ValueError.""" # Drop functions that start with _ very early on here if symbol.ident.startswith('_'): return None - parameters = list(self._create_parameters(symbol.base_type)) + parameters = list(self._create_parameters(symbol, symbol.base_type)) return_ = self._create_return(symbol.base_type.base_type) - try: - name = self._strip_symbol(symbol) - except TransformerException, e: - message.warn_symbol(symbol, e) - return None + name = self._strip_symbol(symbol) func = ast.Function(name, return_, parameters, False, symbol.ident) func.add_symbol_reference(symbol) return func @@ -476,11 +472,11 @@ raise ValueError.""" return value - def _create_parameters(self, base_type): + def _create_parameters(self, symbol, base_type): # warn if we see annotations for unknown parameters param_names = set(child.ident for child in base_type.child_list) - for child in base_type.child_list: - yield self._create_parameter(child) + for i, child in enumerate(base_type.child_list): + yield self._create_parameter(symbol, i, child) def _synthesize_union_type(self, symbol, parent_symbol): # Synthesize a named union so that it can be referenced. @@ -506,13 +502,13 @@ raise ValueError.""" def _create_member(self, symbol, parent_symbol=None): source_type = symbol.base_type - if (source_type.type == CTYPE_POINTER and - symbol.base_type.base_type.type == CTYPE_FUNCTION): + if (source_type.type == CTYPE_POINTER + and symbol.base_type.base_type.type == CTYPE_FUNCTION): node = self._create_callback(symbol, member=True) elif source_type.type == CTYPE_STRUCT and source_type.name is None: - node = self._create_struct(symbol, anonymous=True) + node = self._create_member_compound(ast.Record, symbol) elif source_type.type == CTYPE_UNION and source_type.name is None: - node = self._create_union(symbol, anonymous=True) + node = self._create_member_compound(ast.Union, symbol) else: # Special handling for fields; we don't have annotations on them # to apply later, yet. @@ -523,8 +519,8 @@ raise ValueError.""" # to be able to properly calculate the size of the compound # type (e.g. GValue) that contains this array, see # <https://bugzilla.gnome.org/show_bug.cgi?id=657040>. - if (source_type.base_type.type == CTYPE_UNION and - source_type.base_type.name is None): + if (source_type.base_type.type == CTYPE_UNION + and source_type.base_type.name is None): synthesized_type = self._synthesize_union_type(symbol, parent_symbol) ftype = ast.Array(None, synthesized_type, complete_ctype=complete_ctype) else: @@ -562,27 +558,21 @@ raise ValueError.""" def _create_typedef(self, symbol): ctype = symbol.base_type.type - if (ctype == CTYPE_POINTER and - symbol.base_type.base_type.type == CTYPE_FUNCTION): + if (ctype == CTYPE_POINTER and symbol.base_type.base_type.type == CTYPE_FUNCTION): node = self._create_typedef_callback(symbol) - elif (ctype == CTYPE_POINTER and - symbol.base_type.base_type.type == CTYPE_STRUCT): - node = self._create_typedef_struct(symbol, disguised=True) + elif (ctype == CTYPE_POINTER and symbol.base_type.base_type.type == CTYPE_STRUCT): + node = self._create_typedef_compound(ast.Record, symbol, disguised=True) elif ctype == CTYPE_STRUCT: - node = self._create_typedef_struct(symbol) + node = self._create_typedef_compound(ast.Record, symbol) elif ctype == CTYPE_UNION: - node = self._create_typedef_union(symbol) + node = self._create_typedef_compound(ast.Union, symbol) elif ctype == CTYPE_ENUM: return self._create_enum(symbol) elif ctype in (CTYPE_TYPEDEF, CTYPE_POINTER, CTYPE_BASIC_TYPE, CTYPE_VOID): - try: - name = self.strip_identifier(symbol.ident) - except TransformerException, e: - message.warn(e) - return None + name = self.strip_identifier(symbol.ident) if symbol.base_type.name: complete_ctype = self._create_complete_source_type(symbol.base_type) target = self.create_type_from_ctype_string(symbol.base_type.name, @@ -622,21 +612,6 @@ raise ValueError.""" return canonical - def parse_ctype(self, ctype, is_member=False): - canonical = self._canonicalize_ctype(ctype) - - # Remove all pointers - we require standard calling - # conventions. For example, an 'int' is always passed by - # value (unless it's out or inout). - derefed_typename = canonical.replace('*', '') - - # Preserve "pointerness" of struct/union members - if (is_member and canonical.endswith('*') and - derefed_typename in ast.basic_type_names): - return 'gpointer' - else: - return derefed_typename - def _create_type_from_base(self, source_type, is_parameter=False, is_return=False): ctype = self._create_source_type(source_type) complete_ctype = self._create_complete_source_type(source_type) @@ -693,28 +668,34 @@ raise ValueError.""" return container return ast.Type(ctype=ctype, is_const=is_const, complete_ctype=complete_ctype) - def _create_parameter(self, symbol): + def _create_parameter(self, parent_symbol, index, symbol): if symbol.type == CSYMBOL_TYPE_ELLIPSIS: - ptype = ast.Varargs() + return ast.Parameter('...', ast.Varargs()) else: ptype = self._create_type_from_base(symbol.base_type, is_parameter=True) - return ast.Parameter(symbol.ident, ptype) + + if symbol.ident is None: + if symbol.base_type and symbol.base_type.type != CTYPE_VOID: + message.warn_symbol(parent_symbol, "missing parameter name; undocumentable") + ident = 'arg%d' % (index, ) + else: + ident = symbol.ident + + return ast.Parameter(ident, ptype) def _create_return(self, source_type): typeval = self._create_type_from_base(source_type, is_return=True) return ast.Return(typeval) def _create_const(self, symbol): + if symbol.ident.startswith('_'): + return None + # Don't create constants for non-public things # http://bugzilla.gnome.org/show_bug.cgi?id=572790 - if (symbol.source_filename is None or - not symbol.source_filename.endswith('.h')): - return None - try: - name = self._strip_symbol(symbol) - except TransformerException, e: - message.warn_symbol(symbol, e) + if (symbol.source_filename is None or not symbol.source_filename.endswith('.h')): return None + name = self._strip_symbol(symbol) if symbol.const_string is not None: typeval = ast.TYPE_STRING value = unicode(symbol.const_string, 'utf-8') @@ -731,15 +712,18 @@ raise ValueError.""" if isinstance(target, ast.Type): unaliased = target if unaliased == ast.TYPE_UINT64: - value = str(symbol.const_int % 2**64) + value = str(symbol.const_int % 2 ** 64) elif unaliased == ast.TYPE_UINT32: - value = str(symbol.const_int % 2**32) + value = str(symbol.const_int % 2 ** 32) elif unaliased == ast.TYPE_UINT16: - value = str(symbol.const_int % 2**16) + value = str(symbol.const_int % 2 ** 16) elif unaliased == ast.TYPE_UINT8: - value = str(symbol.const_int % 2**16) + value = str(symbol.const_int % 2 ** 16) else: value = str(symbol.const_int) + elif symbol.const_boolean is not None: + typeval = ast.TYPE_BOOLEAN + value = "true" if symbol.const_boolean else "false" elif symbol.const_double is not None: typeval = ast.TYPE_DOUBLE value = '%f' % (symbol.const_double, ) @@ -751,35 +735,88 @@ raise ValueError.""" const.add_symbol_reference(symbol) return const - def _create_typedef_struct(self, symbol, disguised=False): - try: - name = self.strip_identifier(symbol.ident) - except TransformerException, e: - message.warn_symbol(symbol, e) - return None - struct = ast.Record(name, symbol.ident, disguised=disguised) - self._parse_fields(symbol, struct) - struct.add_symbol_reference(symbol) - self._typedefs_ns[symbol.ident] = struct - return None + def _create_typedef_compound(self, compound_class, symbol, disguised=False): + name = self.strip_identifier(symbol.ident) + assert symbol.base_type + if symbol.base_type.name: + tag_name = symbol.base_type.name + else: + tag_name = None + + # If the struct already exists in the tag namespace, use it. + if tag_name in self._tag_ns: + compound = self._tag_ns[tag_name] + if compound.name: + # If the struct name is set it means the struct has already been + # promoted from the tag namespace to the main namespace by a + # prior typedef struct. If we get here it means this is another + # typedef of that struct. Instead of creating an alias to the + # primary typedef that has been promoted, we create a new Record + # with shared fields. This handles the case where we want to + # give structs like GInitiallyUnowned its own Record: + # typedef struct _GObject GObject; + # typedef struct _GObject GInitiallyUnowned; + # See: http://bugzilla.gnome.org/show_bug.cgi?id=569408 + new_compound = compound_class(name, symbol.ident, tag_name=tag_name) + new_compound.fields = compound.fields + new_compound.add_symbol_reference(symbol) + return new_compound + else: + # If the struct does not have its name set, it exists only in + # the tag namespace. Set it here and return it which will + # promote it to the main namespace. Essentially the first + # typedef for a struct clobbers its name and ctype which is what + # will be visible to GI. + compound.name = name + compound.ctype = symbol.ident + else: + # Create a new struct with a typedef name and tag name when available. + # Structs with a typedef name are promoted into the main namespace + # by it being returned to the "parse" function and are also added to + # the tag namespace if it has a tag_name set. + compound = compound_class(name, symbol.ident, disguised=disguised, tag_name=tag_name) + if tag_name: + # Force the struct as disguised for now since we do not yet know + # if it has fields that will be parsed. Note that this is using + # an erroneous definition of disguised and we should eventually + # only look at the field count when needed. + compound.disguised = True + else: + # Case where we have an anonymous struct which is typedef'd: + # typedef struct {...} Struct; + # we need to parse the fields because we never get a struct + # in the tag namespace which is normally where fields are parsed. + self._parse_fields(symbol, compound) - def _create_typedef_union(self, symbol): - try: - name = self.strip_identifier(symbol.ident) - except TransformerException, e: - message.warn(e) - return None - union = ast.Union(name, symbol.ident) - self._parse_fields(symbol, union) - union.add_symbol_reference(symbol) - self._typedefs_ns[symbol.ident] = union - return None + compound.add_symbol_reference(symbol) + return compound + + def _create_tag_ns_compound(self, compound_class, symbol): + # Get or create a struct from C's tag namespace + if symbol.ident in self._tag_ns: + compound = self._tag_ns[symbol.ident] + else: + compound = compound_class(None, symbol.ident, tag_name=symbol.ident) + + # Make sure disguised is False as we are now about to parse the + # fields of the real struct. + compound.disguised = False + # Fields may need to be parsed in either of the above cases because the + # Record can be created with a typedef prior to the struct definition. + self._parse_fields(symbol, compound) + compound.add_symbol_reference(symbol) + return compound + + def _create_member_compound(self, compound_class, symbol): + compound = compound_class(symbol.ident, symbol.ident) + self._parse_fields(symbol, compound) + compound.add_symbol_reference(symbol) + return compound def _create_typedef_callback(self, symbol): callback = self._create_callback(symbol) if not callback: return None - self._typedefs_ns[callback.name] = callback return callback def _parse_fields(self, symbol, compound): @@ -794,65 +831,21 @@ raise ValueError.""" anonymous_node=child_node) compound.fields.append(field) - def _create_compound(self, klass, symbol, anonymous): - if symbol.ident is None: - # the compound is an anonymous member of another union or a struct - assert anonymous - compound = klass(None, None) - else: - compound = self._typedefs_ns.get(symbol.ident, None) - - if compound is None: - # This is a bit of a hack; really we should try - # to resolve through the typedefs to find the real - # name - if symbol.ident.startswith('_'): - compound = self._typedefs_ns.get(symbol.ident[1:], None) - if compound is None: - if anonymous: - name = symbol.ident - else: - try: - name = self.strip_identifier(symbol.ident) - except TransformerException, e: - message.warn(e) - return None - compound = klass(name, symbol.ident) - - self._parse_fields(symbol, compound) - compound.add_symbol_reference(symbol) - return compound - - def _create_struct(self, symbol, anonymous=False): - return self._create_compound(ast.Record, symbol, anonymous) - - def _create_union(self, symbol, anonymous=False): - return self._create_compound(ast.Union, symbol, anonymous) - def _create_callback(self, symbol, member=False): - parameters = list(self._create_parameters(symbol.base_type.base_type)) + parameters = list(self._create_parameters(symbol, symbol.base_type.base_type)) retval = self._create_return(symbol.base_type.base_type.base_type) # Mark the 'user_data' arguments for i, param in enumerate(parameters): - if (param.type.target_fundamental == 'gpointer' and - param.argname == 'user_data'): + if (param.type.target_fundamental == 'gpointer' and param.argname == 'user_data'): param.closure_name = param.argname if member: name = symbol.ident elif symbol.ident.find('_') > 0: - try: - name = self._strip_symbol(symbol) - except TransformerException, e: - message.warn_symbol(symbol, e) - return None + name = self._strip_symbol(symbol) else: - try: - name = self.strip_identifier(symbol.ident) - except TransformerException, e: - message.warn(e) - return None + name = self.strip_identifier(symbol.ident) callback = ast.Callback(name, retval, parameters, False, ctype=symbol.ident) callback.add_symbol_reference(symbol) @@ -868,9 +861,12 @@ Note that type resolution may not succeed.""" if '.' in typestr: container = self._create_bare_container_type(typestr) if container: - return container - return self._namespace.type_from_name(typestr) - typeval = self.create_type_from_ctype_string(typestr) + typeval = container + else: + typeval = self._namespace.type_from_name(typestr) + else: + typeval = self.create_type_from_ctype_string(typestr) + self.resolve_type(typeval) if typeval.resolved: # Explicitly clear out the c_type; there isn't one in this case. @@ -883,7 +879,7 @@ Note that type resolution may not succeed.""" # which has nominal namespace of "Meta", but a few classes are # "Mutter". We don't export that data in introspection currently. # Basically the library should be fixed, but we'll hack around it here. - for namespace in self._includes.itervalues(): + for namespace in self._parsed_includes.itervalues(): target = namespace.get_by_ctype(pointer_stripped) if target: typeval.target_giname = '%s.%s' % (namespace.name, target.name) @@ -895,9 +891,8 @@ Note that type resolution may not succeed.""" pointer_stripped = typeval.ctype.replace('*', '') try: matches = self.split_ctype_namespaces(pointer_stripped) - except ValueError, e: + except ValueError: return self._resolve_type_from_ctype_all_namespaces(typeval, pointer_stripped) - target_giname = None for namespace, name in matches: target = namespace.get(name) if not target: @@ -916,7 +911,7 @@ Note that type resolution may not succeed.""" return True return False - def resolve_type(self, typeval): + def _resolve_type_internal(self, typeval): if isinstance(typeval, (ast.Array, ast.List)): return self.resolve_type(typeval.element_type) elif isinstance(typeval, ast.Map): @@ -930,28 +925,24 @@ Note that type resolution may not succeed.""" elif typeval.gtype_name: return self._resolve_type_from_gtype_name(typeval) - def _typepair_to_str(self, item): - nsname, item = item - if nsname is None: - return item.name - return '%s.%s' % (nsname, item.name) - - def gtypename_to_giname(self, gtname, names): - resolved = names.type_names.get(gtname) - if resolved: - return self._typepair_to_str(resolved) - resolved = self._names.type_names.get(gtname) - if resolved: - return self._typepair_to_str(resolved) - raise KeyError("Failed to resolve GType name: %r" % (gtname, )) - - def ctype_of(self, obj): - if hasattr(obj, 'ctype'): - return obj.ctype - elif hasattr(obj, 'symbol'): - return obj.symbol - else: - return None + def resolve_type(self, typeval): + if not self._resolve_type_internal(typeval): + return False + + if typeval.target_fundamental or typeval.target_foreign: + return True + + assert typeval.target_giname is not None + + try: + type_ = self.lookup_giname(typeval.target_giname) + except KeyError: + type_ = None + + if type_ is None: + typeval.target_giname = None + + return typeval.resolved def resolve_aliases(self, typenode): """Removes all aliases from typenode, returns first non-alias diff --git a/giscanner/utils.py b/giscanner/utils.py index 642da362..abd1a266 100644 --- a/giscanner/utils.py +++ b/giscanner/utils.py @@ -21,8 +21,12 @@ import re import os import subprocess +import platform + _debugflags = None + + def have_debug_flag(flag): """Check for whether a specific debugging feature is enabled. Well-known flags: @@ -38,6 +42,7 @@ Well-known flags: _debugflags.remove('') return flag in _debugflags + def break_on_debug_flag(flag): if have_debug_flag(flag): import pdb @@ -69,8 +74,10 @@ def to_underscores_noprefix(name): name = _upperstr_pat2.sub(r'\1_\2', name) return name + _libtool_pat = re.compile("dlname='([A-z0-9\.\-\+]+)'\n") + def _extract_dlname_field(la_file): f = open(la_file) data = f.read() @@ -81,6 +88,21 @@ def _extract_dlname_field(la_file): else: return None + +_libtool_libdir_pat = re.compile("libdir='([^']+)'") + + +def _extract_libdir_field(la_file): + f = open(la_file) + data = f.read() + f.close() + m = _libtool_libdir_pat.search(data) + if m: + return m.groups()[0] + else: + return None + + # Returns the name that we would pass to dlopen() the library # corresponding to this .la file def extract_libtool_shlib(la_file): @@ -88,10 +110,19 @@ def extract_libtool_shlib(la_file): if dlname is None: return None + # Darwin uses absolute paths where possible; since the libtool files never + # contain absolute paths, use the libdir field + if platform.system() == 'Darwin': + dlbasename = os.path.basename(dlname) + libdir = _extract_libdir_field(la_file) + if libdir is None: + return dlbasename + return libdir + '/' + dlbasename # From the comments in extract_libtool(), older libtools had # a path rather than the raw dlname return os.path.basename(dlname) + def extract_libtool(la_file): dlname = _extract_dlname_field(la_file) if dlname is None: @@ -104,6 +135,7 @@ def extract_libtool(la_file): libname = libname.replace('.libs/.libs', '.libs') return libname + # Returns arguments for invoking libtool, if applicable, otherwise None def get_libtool_command(options): libtool_infection = not options.nolibtool @@ -118,14 +150,18 @@ def get_libtool_command(options): # we simply split(). return libtool_path.split(' ') + libtool_cmd = 'libtool' + if platform.system() == 'Darwin': + # libtool on OS X is a completely different program written by Apple + libtool_cmd = 'glibtool' try: - subprocess.check_call(['libtool', '--version'], + subprocess.check_call([libtool_cmd, '--version'], stdout=open(os.devnull)) - except (subprocess.CalledProcessError, OSError), e: + except (subprocess.CalledProcessError, OSError): # If libtool's not installed, assume we don't need it return None - return ['libtool'] + return [libtool_cmd] def files_are_identical(path1, path2): @@ -139,3 +175,10 @@ def files_are_identical(path1, path2): f1.close() f2.close() return buf1 == buf2 + + +def cflag_real_include_path(cflag): + if not cflag.startswith("-I"): + return cflag + + return "-I" + os.path.realpath(cflag[2:]) diff --git a/giscanner/xmlwriter.py b/giscanner/xmlwriter.py index fb34adf1..5fdcffed 100755 --- a/giscanner/xmlwriter.py +++ b/giscanner/xmlwriter.py @@ -24,51 +24,11 @@ import os from contextlib import contextmanager from cStringIO import StringIO -from xml.sax.saxutils import escape, quoteattr +from xml.sax.saxutils import escape from .libtoolimporter import LibtoolImporter -def _calc_attrs_length(attributes, indent, self_indent): - if indent == -1: - return -1 - attr_length = 0 - for attr, value in attributes: - # FIXME: actually, if we have attributes with None as value this - # should be considered a bug and raise an error. We are just - # ignoring them here while we fix GIRParser to create the right - # ast with the correct attributes. - if value is None: - continue - attr_length += 2 + len(attr) + len(quoteattr(value)) - return attr_length + indent + self_indent - - -def collect_attributes(tag_name, attributes, self_indent, - self_indent_char, indent=-1): - if not attributes: - return '' - if _calc_attrs_length(attributes, indent, self_indent) > 79: - indent_len = self_indent + len(tag_name) + 1 - else: - indent_len = 0 - first = True - attr_value = '' - for attr, value in attributes: - # FIXME: actually, if we have attributes with None as value this - # should be considered a bug and raise an error. We are just - # ignoring them here while we fix GIRParser to create the right - # ast with the correct attributes. - if value is None: - continue - if indent_len and not first: - attr_value += '\n%s' % (self_indent_char * indent_len) - attr_value += ' %s=%s' % (attr, quoteattr(value)) - if first: - first = False - return attr_value - - with LibtoolImporter(None, None): if 'UNINSTALLED_INTROSPECTION_SRCDIR' in os.environ: from _giscanner import collect_attributes @@ -76,6 +36,25 @@ with LibtoolImporter(None, None): from giscanner._giscanner import collect_attributes +def build_xml_tag(tag_name, attributes=None, data=None, self_indent=0, + self_indent_char=' '): + if attributes is None: + attributes = [] + prefix = u'<%s' % (tag_name, ) + if data is not None: + if isinstance(data, str): + data = data.decode('UTF-8') + suffix = u'>%s</%s>' % (escape(data), tag_name) + else: + suffix = u'/>' + attrs = collect_attributes( + tag_name, attributes, + self_indent, + self_indent_char, + len(prefix) + len(suffix)) + return prefix + attrs + suffix + + class XMLWriter(object): def __init__(self): @@ -91,10 +70,8 @@ class XMLWriter(object): def _open_tag(self, tag_name, attributes=None): if attributes is None: attributes = [] - attrs = collect_attributes( - tag_name, attributes, self._indent, - self._indent_char, - len(tag_name) + 2) + attrs = collect_attributes(tag_name, attributes, + self._indent, self._indent_char, len(tag_name) + 2) self.write_line(u'<%s%s>' % (tag_name, attrs)) def _close_tag(self, tag_name): @@ -118,12 +95,11 @@ class XMLWriter(object): line = line.decode('utf-8') assert isinstance(line, unicode) if do_escape: - line = escape(line.encode('utf-8')).decode('utf-8') + line = escape(line) if indent: - self._data.write('%s%s%s' % ( - self._indent_char * self._indent, - line.encode('utf-8'), - self._newline_char)) + self._data.write('%s%s%s' % (self._indent_char * self._indent, + line.encode('utf-8'), + self._newline_char)) else: self._data.write('%s%s' % (line.encode('utf-8'), self._newline_char)) @@ -131,21 +107,8 @@ class XMLWriter(object): self.write_line('<!-- %s -->' % (text, )) def write_tag(self, tag_name, attributes, data=None): - if attributes is None: - attributes = [] - prefix = u'<%s' % (tag_name, ) - if data is not None: - if isinstance(data, str): - data = data.decode('UTF-8') - suffix = u'>%s</%s>' % (escape(data), tag_name) - else: - suffix = u'/>' - attrs = collect_attributes( - tag_name, attributes, - self._indent, - self._indent_char, - len(prefix) + len(suffix)) - self.write_line(prefix + attrs + suffix) + self.write_line(build_xml_tag(tag_name, attributes, data, + self._indent, self._indent_char)) def push_tag(self, tag_name, attributes=None): if attributes is None: |