QPID-4621: Arithmetic operations for C++ broker selectors

- Rearranged Syntax and parser to support arithmetic
- Implemented arithmetic syntax
- Implemented arithmetic operators
- Added some tests for arithmetic operations
- Added explicit lexical tokens for all the comparison and arithmetic operations
- Added the missing ',' token for us in the 'IN' comparison
- Rearranged the lexer so that everything is not uniformly inside the
  hand crafted state machine.
- Moved a bunch of testing code from the tokeniser to the unit test
  (much of this code might actually not be very useful anymore)

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1457912 13f79535-47bb-0310-9956-ffa450edef68

1 files changed, 153 insertions, 68 deletions

diff --git a/qpid/cpp/src/tests/Selector.cpp b/qpid/cpp/src/tests/Selector.cpp
index 7d36cfe308..7758e3e5b0 100644
--- a/qpid/cpp/src/tests/Selector.cpp
+++ b/qpid/cpp/src/tests/Selector.cpp
@@ -38,14 +38,7 @@ namespace qb = qpid::broker;
 using qpid::broker::Token;
 using qpid::broker::TokenType;
 using qpid::broker::Tokeniser;
-using qpid::broker::tokeniseEos;
-using qpid::broker::tokeniseIdentifier;
-using qpid::broker::tokeniseIdentifierOrReservedWord;
-using qpid::broker::tokeniseReservedWord;
-using qpid::broker::tokeniseOperator;
-using qpid::broker::tokeniseParens;
-using qpid::broker::tokeniseNumeric;
-using qpid::broker::tokeniseString;
+using qpid::broker::tokenise;
 
 namespace qpid {
 namespace tests {
@@ -54,6 +47,88 @@ QPID_AUTO_TEST_SUITE(SelectorSuite)
 
 typedef bool (*TokeniseF)(string::const_iterator&,string::const_iterator&,Token&);
 
+bool tokeniseEos(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
+{
+    Token t1;
+    std::string::const_iterator t = s;
+    bool r = tokenise(t, e, t1);
+    if (r && (t1.type==qb::T_EOS)) {tok = t1; s = t; return true;}
+    return false;
+}
+
+bool tokeniseParens(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
+{
+    Token t1;
+    std::string::const_iterator t = s;
+    bool r = tokenise(t, e, t1);
+    if (r && (t1.type==qb::T_LPAREN || t1.type==qb::T_RPAREN)) {tok = t1; s = t; return true;}
+    return false;
+}
+
+bool tokeniseOperator(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
+{
+    Token t1;
+    std::string::const_iterator t = s;
+    bool r = tokenise(t, e, t1);
+    if (r && (t1.type>=qb::T_PLUS && t1.type<=qb::T_GREQ)) {tok = t1; s = t; return true;}
+    return false;
+}
+
+bool tokeniseString(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
+{
+    Token t1;
+    std::string::const_iterator t = s;
+    bool r = tokenise(t, e, t1);
+    if (r && (t1.type==qb::T_STRING)) {tok = t1; s = t; return true;}
+    return false;
+}
+
+bool tokeniseIdentifier(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
+{
+    Token t1;
+    std::string::const_iterator t = s;
+    bool r = tokenise(t, e, t1);
+    if (r && (t1.type==qb::T_IDENTIFIER)) {tok = t1; s = t; return true;}
+    return false;
+}
+
+bool tokeniseReservedWord(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
+{
+    std::string::const_iterator t = s;
+    Token t1;
+    if (tokenise(t, e, t1)) {
+    switch (t1.type) {
+    case qb::T_AND:
+    case qb::T_BETWEEN:
+    case qb::T_ESCAPE:
+    case qb::T_FALSE:
+    case qb::T_IN:
+    case qb::T_IS:
+    case qb::T_LIKE:
+    case qb::T_NOT:
+    case qb::T_NULL:
+    case qb::T_OR:
+    case qb::T_TRUE:
+        tok = t1;
+        s = t;
+        return true;
+    default:
+        break;
+    }
+    }
+    return false;
+}
+
+bool tokeniseNumeric(std::string::const_iterator& s, std::string::const_iterator& e, Token& tok)
+{
+    Token t1;
+    std::string::const_iterator t = s;
+    bool r = tokenise(t, e, t1);
+    if (r && (t1.type==qb::T_NUMERIC_EXACT || t1.type==qb::T_NUMERIC_APPROX)) {tok = t1; s = t; return true;}
+    return false;
+}
+
+
 void verifyTokeniserSuccess(TokeniseF t, const char* ss, TokenType tt, const char* tv, const char* fs) {
     Token tok;
     string s(ss);
@@ -75,29 +150,29 @@ void verifyTokeniserFail(TokeniseF t, const char* c) {
 
 QPID_AUTO_TEST_CASE(tokeniseSuccess)
 {
-    verifyTokeniserSuccess(&tokeniseEos, "", qb::T_EOS, "", "");
-    verifyTokeniserSuccess(&tokeniseIdentifier, "null_123+blah", qb::T_IDENTIFIER, "null_123", "+blah");
-    verifyTokeniserSuccess(&tokeniseIdentifierOrReservedWord, "null_123+blah", qb::T_IDENTIFIER, "null_123", "+blah");
-    verifyTokeniserSuccess(&tokeniseIdentifierOrReservedWord, "null+blah", qb::T_NULL, "null", "+blah");
-    verifyTokeniserSuccess(&tokeniseIdentifierOrReservedWord, "null+blah", qb::T_NULL, "null", "+blah");
-    verifyTokeniserSuccess(&tokeniseIdentifierOrReservedWord, "Is nOt null", qb::T_IS, "Is", " nOt null");
-    verifyTokeniserSuccess(&tokeniseIdentifierOrReservedWord, "nOt null", qb::T_NOT, "nOt", " null");
-    verifyTokeniserSuccess(&tokeniseIdentifierOrReservedWord, "Is nOt null", qb::T_IS, "Is", " nOt null");
-    verifyTokeniserSuccess(&tokeniseString, "'Hello World'", qb::T_STRING, "Hello World", "");
-    verifyTokeniserSuccess(&tokeniseString, "'Hello World''s end'a bit more", qb::T_STRING, "Hello World's end", "a bit more");
-    verifyTokeniserSuccess(&tokeniseOperator, "=blah", qb::T_OPERATOR, "=", "blah");
-    verifyTokeniserSuccess(&tokeniseOperator, "<> Identifier", qb::T_OPERATOR, "<>", " Identifier");
-    verifyTokeniserSuccess(&tokeniseParens, "(a and b) not c", qb::T_LPAREN, "(", "a and b) not c");
-    verifyTokeniserSuccess(&tokeniseParens, ") not c", qb::T_RPAREN, ")", " not c");
-    verifyTokeniserSuccess(&tokeniseNumeric, "019kill", qb::T_NUMERIC_EXACT, "019", "kill");
-    verifyTokeniserSuccess(&tokeniseNumeric, "0kill", qb::T_NUMERIC_EXACT, "0", "kill");
-    verifyTokeniserSuccess(&tokeniseNumeric, "0.kill", qb::T_NUMERIC_APPROX, "0.", "kill");
-    verifyTokeniserSuccess(&tokeniseNumeric, "3.1415=pi", qb::T_NUMERIC_APPROX, "3.1415", "=pi");
-    verifyTokeniserSuccess(&tokeniseNumeric, ".25.kill", qb::T_NUMERIC_APPROX, ".25", ".kill");
-    verifyTokeniserSuccess(&tokeniseNumeric, "2e5.kill", qb::T_NUMERIC_APPROX, "2e5", ".kill");
-    verifyTokeniserSuccess(&tokeniseNumeric, "3.e50easy to kill", qb::T_NUMERIC_APPROX, "3.e50", "easy to kill");
-    verifyTokeniserSuccess(&tokeniseNumeric, "34.25e+50easy to kill", qb::T_NUMERIC_APPROX, "34.25e+50", "easy to kill");
-    verifyTokeniserSuccess(&tokeniseNumeric, "34.e-50easy to kill", qb::T_NUMERIC_APPROX, "34.e-50", "easy to kill");
+    verifyTokeniserSuccess(&tokenise, "", qb::T_EOS, "", "");
+    verifyTokeniserSuccess(&tokenise, "null_123+blah", qb::T_IDENTIFIER, "null_123", "+blah");
+    verifyTokeniserSuccess(&tokenise, "null_123+blah", qb::T_IDENTIFIER, "null_123", "+blah");
+    verifyTokeniserSuccess(&tokenise, "null+blah", qb::T_NULL, "null", "+blah");
+    verifyTokeniserSuccess(&tokenise, "null+blah", qb::T_NULL, "null", "+blah");
+    verifyTokeniserSuccess(&tokenise, "Is nOt null", qb::T_IS, "Is", " nOt null");
+    verifyTokeniserSuccess(&tokenise, "nOt null", qb::T_NOT, "nOt", " null");
+    verifyTokeniserSuccess(&tokenise, "Is nOt null", qb::T_IS, "Is", " nOt null");
+    verifyTokeniserSuccess(&tokenise, "'Hello World'", qb::T_STRING, "Hello World", "");
+    verifyTokeniserSuccess(&tokenise, "'Hello World''s end'a bit more", qb::T_STRING, "Hello World's end", "a bit more");
+    verifyTokeniserSuccess(&tokenise, "=blah", qb::T_EQUAL, "=", "blah");
+    verifyTokeniserSuccess(&tokenise, "<> Identifier", qb::T_NEQ, "<>", " Identifier");
+    verifyTokeniserSuccess(&tokenise, "(a and b) not c", qb::T_LPAREN, "(", "a and b) not c");
+    verifyTokeniserSuccess(&tokenise, ") not c", qb::T_RPAREN, ")", " not c");
+    verifyTokeniserSuccess(&tokenise, "019kill", qb::T_NUMERIC_EXACT, "019", "kill");
+    verifyTokeniserSuccess(&tokenise, "0kill", qb::T_NUMERIC_EXACT, "0", "kill");
+    verifyTokeniserSuccess(&tokenise, "0.kill", qb::T_NUMERIC_APPROX, "0.", "kill");
+    verifyTokeniserSuccess(&tokenise, "3.1415=pi", qb::T_NUMERIC_APPROX, "3.1415", "=pi");
+    verifyTokeniserSuccess(&tokenise, ".25.kill", qb::T_NUMERIC_APPROX, ".25", ".kill");
+    verifyTokeniserSuccess(&tokenise, "2e5.kill", qb::T_NUMERIC_APPROX, "2e5", ".kill");
+    verifyTokeniserSuccess(&tokenise, "3.e50easy to kill", qb::T_NUMERIC_APPROX, "3.e50", "easy to kill");
+    verifyTokeniserSuccess(&tokenise, "34.25e+50easy to kill", qb::T_NUMERIC_APPROX, "34.25e+50", "easy to kill");
+    verifyTokeniserSuccess(&tokenise, "34.e-50easy to kill", qb::T_NUMERIC_APPROX, "34.e-50", "easy to kill");
 }
 
 QPID_AUTO_TEST_CASE(tokeniseFailure)
@@ -135,7 +210,7 @@ QPID_AUTO_TEST_CASE(tokenString)
     Tokeniser t(s, e);
 
     BOOST_CHECK_EQUAL(t.nextToken(), Token(qb::T_IDENTIFIER, "a"));
-    BOOST_CHECK_EQUAL(t.nextToken(), Token(qb::T_OPERATOR, "="));
+    BOOST_CHECK_EQUAL(t.nextToken(), Token(qb::T_EQUAL, "="));
     BOOST_CHECK_EQUAL(t.nextToken(), Token(qb::T_IDENTIFIER, "b"));
     BOOST_CHECK_EQUAL(t.nextToken(), Token(qb::T_EOS, ""));
 
@@ -146,7 +221,7 @@ QPID_AUTO_TEST_CASE(tokenString)
 
     BOOST_CHECK_EQUAL(u.nextToken(), Token(qb::T_NOT, "not"));
     BOOST_CHECK_EQUAL(u.nextToken(), Token(qb::T_STRING, "hello kitty's friend"));
-    BOOST_CHECK_EQUAL(u.nextToken(), Token(qb::T_OPERATOR, "="));
+    BOOST_CHECK_EQUAL(u.nextToken(), Token(qb::T_EQUAL, "="));
     BOOST_CHECK_EQUAL(u.nextToken(), Token(qb::T_IS, "Is"));
     BOOST_CHECK_EQUAL(u.nextToken(), Token(qb::T_NULL, "null"));
     BOOST_CHECK_EQUAL(u.nextToken(), Token(qb::T_EOS, ""));
@@ -165,12 +240,12 @@ QPID_AUTO_TEST_CASE(tokenString)
 
     BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_LPAREN, "("));
     BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_IDENTIFIER, "a"));
-    BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_OPERATOR, "+"));
+    BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_PLUS, "+"));
     BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_NUMERIC_EXACT, "6"));
     BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_RPAREN, ")"));
-    BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_OPERATOR, "*"));
+    BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_MULT, "*"));
     BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_NUMERIC_APPROX, "7.5"));
-    BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_OPERATOR, "/"));
+    BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_DIV, "/"));
     BOOST_CHECK_EQUAL(v.nextToken(), Token(qb::T_NUMERIC_APPROX, "1e6"));
 }
 
@@ -179,7 +254,6 @@ QPID_AUTO_TEST_CASE(parseStringFail)
     BOOST_CHECK_THROW(qb::Selector e("A is null not"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("A is null or not"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("A is null or and"), std::range_error);
-    BOOST_CHECK_THROW(qb::Selector e("A is null and 'hello out there'"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("A is null and (B='hello out there'"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("in='hello kitty'"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("A like 234"), std::range_error);
@@ -187,38 +261,9 @@ QPID_AUTO_TEST_CASE(parseStringFail)
     BOOST_CHECK_THROW(qb::Selector e("A not like 'eclecti_' escape 'happy'"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("A not like 'eclecti_' escape happy"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("A BETWEEN AND 'true'"), std::range_error);
-    BOOST_CHECK_THROW(qb::Selector e("A NOT BETWEEN (X=Y) AND 3.9"), std::range_error);
     BOOST_CHECK_THROW(qb::Selector e("A NOT BETWEEN 34 OR 3.9"), std::range_error);
 }
 
-class TestSelectorEnv : public qpid::broker::SelectorEnv {
-    mutable map<string, qb::Value> values;
-    boost::ptr_vector<string> strings;
-    static const qb::Value EMPTY;
-
-    const qb::Value& value(const string& v) const {
-        const qb::Value& r = values.find(v)!=values.end() ? values[v] : EMPTY;
-        return r;
-    }
-
-public:
-    void set(const string& id, const char* value) {
-        strings.push_back(new string(value));
-        values[id] = strings[strings.size()-1];
-    }
-
-    void set(const string& id, const qb::Value& value) {
-        if (value.type==qb::Value::T_STRING) {
-            strings.push_back(new string(*value.s));
-            values[id] = strings[strings.size()-1];
-        } else {
-            values[id] = value;
-        }
-    }
-};
-
-const qb::Value TestSelectorEnv::EMPTY;
-
 QPID_AUTO_TEST_CASE(parseString)
 {
     BOOST_CHECK_NO_THROW(qb::Selector e("'Daft' is not null"));
@@ -245,8 +290,43 @@ QPID_AUTO_TEST_CASE(parseString)
     BOOST_CHECK_NO_THROW(qb::Selector e("A LIKE 'excep%ional' EScape '\'"));
     BOOST_CHECK_NO_THROW(qb::Selector e("A BETWEEN 13 AND 'true'"));
     BOOST_CHECK_NO_THROW(qb::Selector e("A NOT BETWEEN 100 AND 3.9"));
+    BOOST_CHECK_NO_THROW(qb::Selector e("true"));
+    BOOST_CHECK_NO_THROW(qb::Selector e("-354"));
+    BOOST_CHECK_NO_THROW(qb::Selector e("-(X or Y)"));
+    BOOST_CHECK_NO_THROW(qb::Selector e("-687 or 567"));
+    BOOST_CHECK_NO_THROW(qb::Selector e("(354.6)"));
+    BOOST_CHECK_NO_THROW(qb::Selector e("A is null and 'hello out there'"));
+    BOOST_CHECK_NO_THROW(qb::Selector e("17/4>4"));
 }
 
+class TestSelectorEnv : public qpid::broker::SelectorEnv {
+    mutable map<string, qb::Value> values;
+    boost::ptr_vector<string> strings;
+    static const qb::Value EMPTY;
+
+    const qb::Value& value(const string& v) const {
+        const qb::Value& r = values.find(v)!=values.end() ? values[v] : EMPTY;
+        return r;
+    }
+
+public:
+    void set(const string& id, const char* value) {
+        strings.push_back(new string(value));
+        values[id] = strings[strings.size()-1];
+    }
+
+    void set(const string& id, const qb::Value& value) {
+        if (value.type==qb::Value::T_STRING) {
+            strings.push_back(new string(*value.s));
+            values[id] = strings[strings.size()-1];
+        } else {
+            values[id] = value;
+        }
+    }
+};
+
+const qb::Value TestSelectorEnv::EMPTY;
+
 QPID_AUTO_TEST_CASE(simpleEval)
 {
     TestSelectorEnv env;
@@ -296,13 +376,18 @@ QPID_AUTO_TEST_CASE(numericEval)
     BOOST_CHECK(qb::Selector("B=39.0").eval(env));
     BOOST_CHECK(qb::Selector("Not A=17 or B=5.6").eval(env));
     BOOST_CHECK(!qb::Selector("A<>17 and B=5.6e17").eval(env));
+    BOOST_CHECK(qb::Selector("3 BETWEEN -17 and 98.5").eval(env));
     BOOST_CHECK(qb::Selector("A BETWEEN B and 98.5").eval(env));
     BOOST_CHECK(!qb::Selector("B NOT BETWEEN 35 AND 100").eval(env));
     BOOST_CHECK(!qb::Selector("A BETWEEN B and 40").eval(env));
     BOOST_CHECK(!qb::Selector("A BETWEEN C and 40").eval(env));
     BOOST_CHECK(!qb::Selector("A BETWEEN 45 and C").eval(env));
-    BOOST_CHECK(!qb::Selector("A BETWEEN 40 and C OR A NOT BETWEEN 40 and C").eval(env));
-    BOOST_CHECK(!qb::Selector("A BETWEEN C and 45 OR A NOT BETWEEN C and 45").eval(env));
+    BOOST_CHECK(qb::Selector("(A BETWEEN 40 and C) IS NULL").eval(env));
+    BOOST_CHECK(qb::Selector("(A BETWEEN C and 45) IS NULL").eval(env));
+    BOOST_CHECK(qb::Selector("17/4=4").eval(env));
+    BOOST_CHECK(!qb::Selector("A/0=0").eval(env));
+    BOOST_CHECK(qb::Selector("A*B+19<A*(B+19)").eval(env));
+    BOOST_CHECK(qb::Selector("-A=0-A").eval(env));
 }
 
 QPID_AUTO_TEST_CASE(comparisonEval)