rename PythonEditor plugin to Python

The plugin does not only contain a pure editor, but all kind of support
for a programming language like project and run support.

Change-Id: I1251367c8db2e7a54986415ffc5b860cb210de3c
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

1 files changed, 392 insertions, 0 deletions

diff --git a/src/plugins/python/pythonscanner.cpp b/src/plugins/python/pythonscanner.cpp
new file mode 100644
index 0000000000..abb6251e15
--- /dev/null
+++ b/src/plugins/python/pythonscanner.cpp
@@ -0,0 +1,392 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "pythonscanner.h"
+
+#include "pythonconstants.h"
+#include "pythonplugin.h"
+
+namespace Python {
+namespace Internal {
+
+Scanner::Scanner(const QChar *text, const int length)
+    : m_text(text), m_textLength(length), m_state(0)
+{
+}
+
+void Scanner::setState(int state)
+{
+    m_state = state;
+}
+
+int Scanner::state() const
+{
+    return m_state;
+}
+
+FormatToken Scanner::read()
+{
+    setAnchor();
+    if (isEnd())
+        return FormatToken();
+
+    State state;
+    QChar saved;
+    parseState(state, saved);
+    switch (state) {
+    case State_String:
+        return readStringLiteral(saved);
+    case State_MultiLineString:
+        return readMultiLineStringLiteral(saved);
+    default:
+        return onDefaultState();
+    }
+}
+
+QString Scanner::value(const FormatToken &tk) const
+{
+    return QString(m_text + tk.begin(), tk.length());
+}
+
+FormatToken Scanner::onDefaultState()
+{
+    QChar first = peek();
+    move();
+
+    if (first == '\\' && peek() == '\n') {
+        move();
+        return FormatToken(Format_Whitespace, anchor(), 2);
+    }
+
+    if (first == '.' && peek().isDigit())
+        return readFloatNumber();
+
+    if (first == '\'' || first == '\"')
+        return readStringLiteral(first);
+
+    if (first.isLetter() || first == '_')
+        return readIdentifier();
+
+    if (first.isDigit())
+        return readNumber();
+
+    if (first == '#') {
+        if (peek() == '#')
+            return readDoxygenComment();
+        return readComment();
+    }
+
+    if (first.isSpace())
+        return readWhiteSpace();
+
+    return readOperator();
+}
+
+/**
+ * @brief Lexer::passEscapeCharacter
+ * @return returns true if escape sequence doesn't end with newline
+ */
+void Scanner::checkEscapeSequence(QChar quoteChar)
+{
+    if (peek() == '\\') {
+        move();
+        QChar ch = peek();
+        if (ch == '\n' || ch.isNull())
+            saveState(State_String, quoteChar);
+    }
+}
+
+/**
+  reads single-line string literal, surrounded by ' or " quotes
+  */
+FormatToken Scanner::readStringLiteral(QChar quoteChar)
+{
+    QChar ch = peek();
+    if (ch == quoteChar && peek(1) == quoteChar) {
+        saveState(State_MultiLineString, quoteChar);
+        return readMultiLineStringLiteral(quoteChar);
+    }
+
+    while (ch != quoteChar && !ch.isNull()) {
+        checkEscapeSequence(quoteChar);
+        move();
+        ch = peek();
+    }
+    if (ch == quoteChar)
+        clearState();
+    move();
+    return FormatToken(Format_String, anchor(), length());
+}
+
+/**
+  reads multi-line string literal, surrounded by ''' or """ sequences
+  */
+FormatToken Scanner::readMultiLineStringLiteral(QChar quoteChar)
+{
+    for (;;) {
+        QChar ch = peek();
+        if (ch.isNull())
+            break;
+        if (ch == quoteChar && peek(1) == quoteChar && peek(2) == quoteChar) {
+            clearState();
+            move();
+            move();
+            move();
+            break;
+        }
+        move();
+    }
+
+    return FormatToken(Format_String, anchor(), length());
+}
+
+/**
+  reads identifier and classifies it
+  */
+FormatToken Scanner::readIdentifier()
+{
+    static const QSet<QString> keywords = {
+        "and", "as", "assert", "break", "class", "continue", "def", "del", "elif",
+        "else", "except", "exec", "finally", "for", "from", "global", "if", "import",
+        "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try",
+        "while", "with", "yield"
+    };
+
+    // List of Python magic methods and attributes
+    static const QSet<QString> magics = {
+        // ctor & dtor
+        "__init__", "__del__",
+        // string conversion functions
+        "__str__", "__repr__", "__unicode__",
+        // attribute access functions
+        "__setattr__", "__getattr__", "__delattr__",
+        // binary operators
+        "__add__", "__sub__", "__mul__", "__truediv__", "__floordiv__", "__mod__",
+        "__pow__", "__and__", "__or__", "__xor__", "__eq__", "__ne__", "__gt__",
+        "__lt__", "__ge__", "__le__", "__lshift__", "__rshift__", "__contains__",
+        // unary operators
+        "__pos__", "__neg__", "__inv__", "__abs__", "__len__",
+        // item operators like []
+        "__getitem__", "__setitem__", "__delitem__", "__getslice__", "__setslice__",
+        "__delslice__",
+        // other functions
+        "__cmp__", "__hash__", "__nonzero__", "__call__", "__iter__", "__reversed__",
+        "__divmod__", "__int__", "__long__", "__float__", "__complex__", "__hex__",
+        "__oct__", "__index__", "__copy__", "__deepcopy__", "__sizeof__", "__trunc__",
+        "__format__",
+        // magic attributes
+        "__name__", "__module__", "__dict__", "__bases__", "__doc__"
+    };
+
+    // List of python built-in functions and objects
+    static const QSet<QString> builtins = {
+        "range", "xrange", "int", "float", "long", "hex", "oct", "chr", "ord",
+        "len", "abs", "None", "True", "False"
+    };
+
+    QChar ch = peek();
+    while (ch.isLetterOrNumber() || ch == '_') {
+        move();
+        ch = peek();
+    }
+
+    const QString v = QString(m_text + m_markedPosition, length());
+    Format tkFormat = Format_Identifier;
+    if (v == "self")
+        tkFormat = Format_ClassField;
+    else if (builtins.contains(v))
+        tkFormat = Format_Type;
+    else if (magics.contains(v))
+        tkFormat = Format_MagicAttr;
+    else if (keywords.contains(v))
+        tkFormat = Format_Keyword;
+
+    return FormatToken(tkFormat, anchor(), length());
+}
+
+inline static bool isHexDigit(QChar ch)
+{
+    return ch.isDigit()
+            || (ch >= 'a' && ch <= 'f')
+            || (ch >= 'A' && ch <= 'F');
+}
+
+inline static bool isOctalDigit(QChar ch)
+{
+    return ch.isDigit() && ch != '8' && ch != '9';
+}
+
+inline static bool isBinaryDigit(QChar ch)
+{
+    return ch == '0' || ch == '1';
+}
+
+inline static bool isValidIntegerSuffix(QChar ch)
+{
+    return ch == 'l' || ch == 'L';
+}
+
+FormatToken Scanner::readNumber()
+{
+    if (!isEnd()) {
+        QChar ch = peek();
+        if (ch.toLower() == 'b') {
+            move();
+            while (isBinaryDigit(peek()))
+                move();
+        } else if (ch.toLower() == 'o') {
+            move();
+            while (isOctalDigit(peek()))
+                move();
+        } else if (ch.toLower() == 'x') {
+            move();
+            while (isHexDigit(peek()))
+                move();
+        } else { // either integer or float number
+            return readFloatNumber();
+        }
+        if (isValidIntegerSuffix(peek()))
+            move();
+    }
+    return FormatToken(Format_Number, anchor(), length());
+}
+
+FormatToken Scanner::readFloatNumber()
+{
+    enum
+    {
+        State_INTEGER,
+        State_FRACTION,
+        State_EXPONENT
+    } state;
+    state = (peek(-1) == '.') ? State_FRACTION : State_INTEGER;
+
+    for (;;) {
+        QChar ch = peek();
+        if (ch.isNull())
+            break;
+
+        if (state == State_INTEGER) {
+            if (ch == '.')
+                state = State_FRACTION;
+            else if (!ch.isDigit())
+                break;
+        } else if (state == State_FRACTION) {
+            if (ch == 'e' || ch == 'E') {
+                QChar next = peek(1);
+                QChar next2 = peek(2);
+                bool isExp = next.isDigit()
+                        || ((next == '-' || next == '+') && next2.isDigit());
+                if (isExp) {
+                    move();
+                    state = State_EXPONENT;
+                } else {
+                    break;
+                }
+            } else if (!ch.isDigit()) {
+                break;
+            }
+        } else if (!ch.isDigit()) {
+            break;
+        }
+        move();
+    }
+
+    QChar ch = peek();
+    if ((state == State_INTEGER && (ch == 'l' || ch == 'L'))
+            || (ch == 'j' || ch =='J'))
+        move();
+
+    return FormatToken(Format_Number, anchor(), length());
+}
+
+/**
+  reads single-line python comment, started with "#"
+  */
+FormatToken Scanner::readComment()
+{
+    QChar ch = peek();
+    while (ch != '\n' && !ch.isNull()) {
+        move();
+        ch = peek();
+    }
+    return FormatToken(Format_Comment, anchor(), length());
+}
+
+/**
+  reads single-line python doxygen comment, started with "##"
+  */
+FormatToken Scanner::readDoxygenComment()
+{
+    QChar ch = peek();
+    while (ch != '\n' && !ch.isNull()) {
+        move();
+        ch = peek();
+    }
+    return FormatToken(Format_Doxygen, anchor(), length());
+}
+
+/**
+  reads whitespace
+  */
+FormatToken Scanner::readWhiteSpace()
+{
+    while (peek().isSpace())
+        move();
+    return FormatToken(Format_Whitespace, anchor(), length());
+}
+
+/**
+  reads punctuation symbols, excluding some special
+  */
+FormatToken Scanner::readOperator()
+{
+    static const QString EXCLUDED_CHARS = "\'\"_#";
+    QChar ch = peek();
+    while (ch.isPunct() && !EXCLUDED_CHARS.contains(ch)) {
+        move();
+        ch = peek();
+    }
+    return FormatToken(Format_Operator, anchor(), length());
+}
+
+void Scanner::clearState()
+{
+    m_state = 0;
+}
+
+void Scanner::saveState(State state, QChar savedData)
+{
+    m_state = (state << 16) | static_cast<int>(savedData.unicode());
+}
+
+void Scanner::parseState(State &state, QChar &savedData) const
+{
+    state = static_cast<State>(m_state >> 16);
+    savedData = static_cast<ushort>(m_state);
+}
+
+} // namespace Internal
+} // namespace Python