eolian: initial lexing of documentation

3 files changed, 155 insertions, 12 deletions

diff --git a/src/lib/eolian/eo_lexer.c b/src/lib/eolian/eo_lexer.c
index 32be9f90fa..0e9548f3cc 100644
--- a/src/lib/eolian/eo_lexer.c
+++ b/src/lib/eolian/eo_lexer.c
@@ -49,7 +49,7 @@ static const char * const tokens[] =
 {
    "==", "!=", ">=", "<=", "&&", "||", "<<", ">>",
 
-   "<comment>", "<string>", "<char>", "<number>", "<value>",
+   "<comment>", "<doc>", "<string>", "<char>", "<number>", "<value>",
 
    KEYWORDS
 };
@@ -114,7 +114,7 @@ init_hash(void)
    unsigned int i, u;
    if (keyword_map) return;
    keyword_map = eina_hash_string_superfast_new(NULL);
-   for (i = u = 13; i < (sizeof(tokens) / sizeof(const char*)); ++i)
+   for (i = u = 14; i < (sizeof(tokens) / sizeof(const char*)); ++i)
      {
          eina_hash_add(keyword_map, tokens[i], (void*)(size_t)(i - u + 1));
      }
@@ -253,6 +253,107 @@ cend:
 }
 
 static void
+read_doc(Eo_Lexer *ls, Eo_Token *tok)
+{
+   Eo_Doc *doc = calloc(1, sizeof(Eo_Doc));
+   eina_strbuf_reset(ls->buff);
+
+   skip_ws(ls);
+   while (is_newline(ls->current))
+     next_line_ws(ls);
+
+   for (;;)
+     {
+        if (!ls->current)
+          {
+             free(doc);
+             eo_lexer_lex_error(ls, "unfinished documentation", -1);
+          }
+        if (is_newline(ls->current))
+          {
+             while (is_newline(ls->current))
+               next_line_ws(ls);
+             break;
+          }
+        else
+          {
+             if (ls->current == ']')
+               {
+                  next_char(ls);
+                  if (ls->current != ']')
+                    eina_strbuf_append_char(ls->buff, ']');
+                  else
+                    {
+                       next_char(ls);
+                       eina_strbuf_trim(ls->buff);
+                       doc->summary = eina_stringshare_add(
+                           eina_strbuf_string_get(ls->buff));
+                       tok->value.doc = doc;
+                       return;
+                    }
+               }
+             eina_strbuf_append_char(ls->buff, ls->current);
+             next_char(ls);
+          }
+     }
+
+   eina_strbuf_trim(ls->buff);
+   doc->summary = eina_stringshare_add(eina_strbuf_string_get(ls->buff));
+
+   Eina_Strbuf *rbuf = eina_strbuf_new();
+   for (;;)
+     {
+        if (!ls->current)
+          {
+             eina_stringshare_del(doc->summary);
+             free(doc);
+             eina_strbuf_free(rbuf);
+             eo_lexer_lex_error(ls, "unfinished documentation", -1);
+          }
+
+        eina_strbuf_reset(ls->buff);
+        while (ls->current && !is_newline(ls->current))
+          {
+             if (ls->current == ']')
+               {
+                  next_char(ls);
+                  if (ls->current != ']')
+                    eina_strbuf_append_char(ls->buff, ']');
+                  else
+                    {
+                       next_char(ls);
+                       eina_strbuf_trim(ls->buff);
+                       eina_strbuf_append(rbuf, eina_strbuf_string_get(ls->buff));
+                       eina_strbuf_trim(rbuf);
+                       if (eina_strbuf_string_get(rbuf)[0])
+                         doc->description = eina_stringshare_add(
+                             eina_strbuf_string_get(rbuf));
+                       eina_strbuf_free(rbuf);
+                       tok->value.doc = doc;
+                       return;
+                    }
+               }
+             eina_strbuf_append_char(ls->buff, ls->current);
+             next_char(ls);
+          }
+        eina_strbuf_trim(ls->buff);
+        eina_strbuf_append(rbuf, eina_strbuf_string_get(ls->buff));
+
+        if (is_newline(ls->current))
+          {
+             next_line_ws(ls);
+             /* new paragraph */
+             if (is_newline(ls->current))
+               eina_strbuf_append(rbuf, "\n\n");
+             else
+               eina_strbuf_append_char(rbuf, ' ');
+             while (is_newline(ls->current))
+               next_line_ws(ls);
+          }
+     }
+}
+
+static void
 esc_error(Eo_Lexer *ls, int *c, int n, const char *msg)
 {
    int i;
@@ -549,6 +650,12 @@ lex(Eo_Lexer *ls, Eo_Token *tok)
              }
            continue;
         }
+      case '[':
+        next_char(ls);
+        if (ls->current != '[') return '[';
+        next_char(ls);
+        read_doc(ls, tok);
+        return TOK_DOC;
       case '\0':
         return -1;
       case '=':
@@ -748,6 +855,25 @@ _temps_free(Eo_Lexer_Temps *tmp)
      if (s) eina_stringshare_del(s);
 }
 
+static void
+_free_tok(Eo_Token *tok)
+{
+   if (tok->token < START_CUSTOM || tok->token == TOK_NUMBER ||
+                                    tok->token == TOK_CHAR)
+     return;
+   if (tok->token == TOK_DOC)
+     {
+        /* free doc */
+        eina_stringshare_del(tok->value.doc->summary);
+        eina_stringshare_del(tok->value.doc->description);
+        free(tok->value.doc);
+        tok->value.doc = NULL;
+        return;
+     }
+   eina_stringshare_del(tok->value.s);
+   tok->value.s = NULL;
+}
+
 void
 eo_lexer_free(Eo_Lexer *ls)
 {
@@ -757,6 +883,7 @@ eo_lexer_free(Eo_Lexer *ls)
    if (ls->buff    ) eina_strbuf_free    (ls->buff);
    if (ls->handle  ) eina_file_close     (ls->handle);
 
+   _free_tok(&ls->t);
    eo_lexer_context_clear(ls);
    _temps_free(&ls->tmp);
    free(ls);
@@ -778,12 +905,7 @@ eo_lexer_new(const char *source)
 int
 eo_lexer_get(Eo_Lexer *ls)
 {
-   if (ls->t.token >= START_CUSTOM && ls->t.token != TOK_NUMBER
-                                   && ls->t.token != TOK_CHAR)
-     {
-        eina_stringshare_del(ls->t.value.s);
-        ls->t.value.s = NULL;
-     }
+   _free_tok(&ls->t);
    if (ls->lookahead.token >= 0)
      {
         ls->t               = ls->lookahead;
@@ -850,7 +972,7 @@ eo_lexer_token_to_str(int token, char *buf)
 const char *
 eo_lexer_keyword_str_get(int kw)
 {
-   return tokens[kw + 12];
+   return tokens[kw + 13];
 }
 
 Eina_Bool
diff --git a/src/lib/eolian/eo_lexer.h b/src/lib/eolian/eo_lexer.h
index aea3a1c839..f56bc0b61b 100644
--- a/src/lib/eolian/eo_lexer.h
+++ b/src/lib/eolian/eo_lexer.h
@@ -17,7 +17,7 @@ enum Tokens
    TOK_EQ = START_CUSTOM, TOK_NQ, TOK_GE, TOK_LE,
    TOK_AND, TOK_OR, TOK_LSH, TOK_RSH,
 
-   TOK_COMMENT, TOK_STRING, TOK_CHAR, TOK_NUMBER, TOK_VALUE
+   TOK_COMMENT, TOK_DOC, TOK_STRING, TOK_CHAR, TOK_NUMBER, TOK_VALUE
 };
 
 /* all keywords in eolian, they can still be used as names (they're TOK_VALUE)
@@ -81,13 +81,34 @@ enum Numbers
    NUM_DOUBLE
 };
 
+typedef struct _Eo_Doc
+{
+   Eina_Stringshare *summary;
+   Eina_Stringshare *description;
+} Eo_Doc;
+
+typedef union
+{
+   char               c;
+   const    char     *s;
+   signed   int       i;
+   unsigned int       u;
+   signed   long      l;
+   unsigned long      ul;
+   signed   long long ll;
+   unsigned long long ull;
+   float              f;
+   double             d;
+   Eo_Doc            *doc;
+} Eo_Token_Union;
+
 /* a token - "token" is the actual token id, "value" is the value of a token
  * if needed - NULL otherwise - for example the value of a TOK_VALUE, "kw"
  * is the keyword id if this is a keyword, it's 0 when not a keyword */
 typedef struct _Eo_Token
 {
    int token, kw;
-   Eolian_Value_Union value;
+   Eo_Token_Union value;
 } Eo_Token;
 
 typedef struct _Lexer_Ctx
diff --git a/src/lib/eolian/eo_parser.c b/src/lib/eolian/eo_parser.c
index 88d1a1e6ab..6a34d2c90c 100644
--- a/src/lib/eolian/eo_parser.c
+++ b/src/lib/eolian/eo_parser.c
@@ -326,7 +326,7 @@ parse_expr_simple(Eo_Lexer *ls)
            expr = push_expr(ls);
            FILL_BASE(expr->base, ls, line, col);
            expr->type = ls->t.kw + 1; /* map Numbers from lexer to expr type */
-           expr->value = ls->t.value;
+           memcpy(&expr->value, &ls->t.value, sizeof(expr->value));
            eo_lexer_get(ls);
            break;
         }