docgen: migrate to new Eolian-provided tokenizer

4 files changed, 99 insertions, 180 deletions

diff --git a/src/bindings/luajit/eolian.lua b/src/bindings/luajit/eolian.lua
index c5e2e79d29..66727fa175 100644
--- a/src/bindings/luajit/eolian.lua
+++ b/src/bindings/luajit/eolian.lua
@@ -1551,7 +1551,7 @@ M.documentation_string_split = function(str)
     local sep = str:find("\n\n", 1, true)
     local ret = {}
     while true do
-        local pstr = (sep and str:sub(1, sep - 1) or pstr):match("^%s*(.-)%s*$")
+        local pstr = (sep and str:sub(1, sep - 1) or str):match("^%s*(.-)%s*$")
         if #pstr > 0 then
             ret[#ret + 1] = pstr
         end
diff --git a/src/scripts/elua/apps/docgen/doctree.lua b/src/scripts/elua/apps/docgen/doctree.lua
index da1fd8827f..7c86d34760 100644
--- a/src/scripts/elua/apps/docgen/doctree.lua
+++ b/src/scripts/elua/apps/docgen/doctree.lua
@@ -43,7 +43,7 @@ local gen_doc_refd = function(str)
     if not str then
         return nil
     end
-    local pars = dutil.str_split(str, "\n\n")
+    local pars = eolian.documentation_string_split(str)
     for i = 1, #pars do
         pars[i] = writer.Buffer():write_par(pars[i]):finish()
     end
@@ -1269,94 +1269,76 @@ M.Expression = Node:clone {
     end
 }
 
-local decl_to_nspace = function(decl)
-    local dt = eolian.declaration_type
-    local decltypes = {
-        [dt.ALIAS] = "alias",
-        [dt.STRUCT] = "struct",
-        [dt.ENUM] = "enum",
-        [dt.VAR] = "var"
-    }
-    local ns = decltypes[decl:type_get()]
-    if ns then
-        return ns
-    elseif decl:type_get() == dt.CLASS then
-        local ret = M.Class(decl:class_get()):type_str_get()
-        if not ret then
-            error("unknown class type for class '" .. decl:name_get() .. "'")
-        end
-        return ret
-    else
-        error("unknown declaration type for declaration '"
-            .. decl:name_get() .. "'")
-    end
-end
-
-M.ref_get = function(str, root)
-    local decl = eolian.declaration_get_by_name(str)
-    if decl then
-        local t = { decl_to_nspace(decl) }
-        for tok in str:gmatch("[^%.]+") do
-            t[#t + 1] = tok:lower()
-        end
-        if root ~= nil then t[#t + 1] = not not root end
-        return t
-    end
+M.DocTokenizer = Node:clone {
+    UNKNOWN          = eolian.doc_token_type.UNKNOWN,
+    TEXT             = eolian.doc_token_type.TEXT,
+    REF              = eolian.doc_token_type.REF,
+    MARK_NOTE        = eolian.doc_token_type.MARK_NOTE,
+    MARK_WARNING     = eolian.doc_token_type.MARK_WARNING,
+    MARK_REMARK      = eolian.doc_token_type.MARK_REMARK,
+    MARK_TODO        = eolian.doc_token_type.MARK_TODO,
+    MARKUP_MONOSPACE = eolian.doc_token_type.MARKUP_MONOSPACE,
 
-    -- field or func
-    local bstr = str:match("(.+)%.[^.]+")
-    if not bstr then
-        error("invalid reference '" .. str .. "'")
-    end
+    __ctor = function(self, str)
+        self.tok = eolian.doc_token_init()
+        self.str = str
+        assert(self.str)
+        assert(self.tok)
+    end,
 
-    local sfx = str:sub(#bstr + 1)
+    tokenize = function(self)
+       self.str = eolian.documentation_tokenize(self.str, self.tok)
+       return not not self.str
+    end,
 
-    decl = eolian.declaration_get_by_name(bstr)
-    if decl then
-        local dt = eolian.declaration_type
-        local tp = decl:type_get()
-        if tp == dt.STRUCT or tp == dt.ENUM then
-            -- TODO: point to the actual item
-            return M.ref_get(bstr, root)
-        end
-    end
+    text_get = function(self)
+        return self.tok:text_get()
+    end,
 
-    local cl = M.Class.by_name_get(bstr)
-    local fn
-    local ftype = M.Function.UNRESOLVED
-    if not cl then
-        if sfx == ".get" then
-            ftype = M.Function.PROP_GET
-        elseif sfx == ".set" then
-            ftype = M.Function.PROP_SET
-        end
-        local mname
-        if ftype ~= M.Function.UNRESOLVED then
-            mname = bstr:match(".+%.([^.]+)")
-            if not mname then
-                error("invalid reference '" .. str .. "'")
-            end
-            bstr = bstr:match("(.+)%.[^.]+")
-            cl = M.Class.by_name_get(bstr)
-            if cl then
-                fn = cl:function_get_by_name(mname, ftype)
+    type_get = function(self)
+        return self.tok:type_get()
+    end,
+
+    ref_get = function(self, root)
+        local tp, d1, d2 = self.tok:ref_get()
+        local reft = eolian.doc_ref_type
+        local ret
+        if tp == reft.CLASS or tp == reft.FUNC or tp == reft.EVENT then
+            ret = { M.Class(d1):type_str_get() }
+            if not ret[1] then
+                error("unknown class type for class '"
+                      .. d1:full_name_get() .. "'")
             end
+        elseif tp == reft.ALIAS then
+            ret = { "alias" }
+        elseif tp == reft.STRUCT or tp == reft.STRUCT_FIELD then
+            -- TODO: point to field
+            ret = { "struct" }
+        elseif tp == reft.ENUM or tp == reft.ENUM_FIELD  then
+            -- TODO: point to field
+            ret = { "enum" }
+        elseif tp == reft.VAR then
+            ret = { "var" }
+        else
+            error("invalid reference '" .. self:text_get() .. "'")
         end
-    else
-        fn = cl:function_get_by_name(sfx:sub(2), ftype)
-        if fn then ftype = fn:type_get() end
-    end
-
-    if not fn or not fn:type_str_get() then
-        error("invalid reference '" .. str .. "'")
+        for tok in d1:full_name_get():gmatch("[^%.]+") do
+            ret[#ret + 1] = tok:lower()
+        end
+        if tp == reft.FUNC then
+            local fid = M.Function(d2)
+            ret[#ret + 1] = fid:type_str_get()
+            ret[#ret + 1] = fid:name_get():lower()
+        elseif tp == reft.EVENT then
+            ret[#ret + 1] = "event"
+            ret[#ret + 1] = d2:name_get():lower()
+        end
+        if root ~= nil then
+            ret[#ret + 1] = not not root
+        end
+        return ret
     end
-
-    local ret = M.ref_get(bstr)
-    ret[#ret + 1] = fn:type_str_get()
-    ret[#ret + 1] = fn:name_get():lower()
-    if root ~= nil then ret[#ret + 1] = not not root end
-    return ret
-end
+}
 
 M.scan_directory = function(dir)
     if not dir then
diff --git a/src/scripts/elua/apps/docgen/util.lua b/src/scripts/elua/apps/docgen/util.lua
index b8a0aa2da2..7e14a37b6d 100644
--- a/src/scripts/elua/apps/docgen/util.lua
+++ b/src/scripts/elua/apps/docgen/util.lua
@@ -42,26 +42,6 @@ M.rm_root = function()
     cutil.file_rmrf(M.path_join(doc_root, "auto"))
 end
 
-M.str_split = function(str, delim)
-    if not str then
-        return nil
-    end
-    local s, e = str:find(delim, 1, true)
-    if not s then
-        return { str }
-    end
-    local t = {}
-    while s do
-        t[#t + 1] = str:sub(1, s - 1)
-        str = str:sub(e + 1)
-        s, e = str:find(delim, 1, true)
-        if not s then
-            t[#t + 1] = str
-        end
-    end
-    return t
-end
-
 M.init = function(root, rns)
     doc_root = root:gsub(rep_sep, path_sep)
     root_ns = rns
diff --git a/src/scripts/elua/apps/docgen/writer.lua b/src/scripts/elua/apps/docgen/writer.lua
index 9ab53cdec8..167d77fef0 100644
--- a/src/scripts/elua/apps/docgen/writer.lua
+++ b/src/scripts/elua/apps/docgen/writer.lua
@@ -359,93 +359,50 @@ M.Writer = util.Object:clone {
         return self
     end,
 
-    write_par_markup = function(self, str)
-        self:write_raw("%%")
-        local f = str:gmatch(".")
-        local c = f()
-        while c do
-            if c == "\\" then
-                c = f()
-                if c ~= "@" and c ~= "$" then
-                    self:write_raw("\\")
-                end
-                self:write_raw(c)
-                c = f()
-            elseif c == "$" then
-                c = f()
-                if c and c:match("[a-zA-Z_]") then
-                    local wbuf = { c }
-                    c = f()
-                    while c and c:match("[a-zA-Z0-9_]") do
-                        wbuf[#wbuf + 1] = c
-                        c = f()
-                    end
-                    self:write_raw("%%''" .. table.concat(wbuf) .. "''%%")
-                else
-                    self:write_raw("$")
+    write_par = function(self, str)
+        local tokp = dtree.DocTokenizer(str)
+        local notetypes = M.has_feature("notes") and {
+            [tokp.MARK_NOTE] = "<note>\n",
+            [tokp.MARK_WARNING] = "<note warning>\n",
+            [tokp.MARK_REMARK] = "<note tip>\n",
+            [tokp.MARK_TODO] = "<note>\n**TODO:** "
+        } or {}
+        local hasraw, hasnote = false, false
+        while tokp:tokenize() do
+            local tp = tokp:type_get()
+            if notetypes[tp] then
+                self:write_raw(tag)
+                hasnote = true
+            else
+                if not hasraw then
+                    self:write_raw("%%")
+                    hasraw = true
                 end
-            elseif c == "@" then
-                c = f()
-                if c and c:match("[a-zA-Z_]") then
-                    local rbuf = { c }
-                    c = f()
-                    while c and c:match("[a-zA-Z0-9_.]") do
-                        rbuf[#rbuf + 1] = c
-                        c = f()
+                if tp == tokp.REF then
+                    local reft = tokp:ref_get(true)
+                    local str = tokp:text_get()
+                    if str:sub(1, 1) == "[" then
+                        str = str:sub(2, #str - 1)
                     end
-                    local ldot = false
-                    if rbuf[#rbuf] == "." then
-                        ldot = true
-                        rbuf[#rbuf] = nil
-                    end
-                    local title = table.concat(rbuf)
                     self:write_raw("%%")
-                    self:write_link(dtree.ref_get(title, true), title)
+                    self:write_link(reft, str)
                     self:write_raw("%%")
-                    if ldot then
-                        self:write_raw(".")
-                    end
-                else
-                    self:write_raw("@")
-                end
-            elseif c == "%" then
-                c = f()
-                if c == "%" then
-                    c = f()
-                    self:write_raw("%%<nowiki>%%</nowiki>%%")
                 else
-                    self:write_raw("%")
+                    local str = tokp:text_get()
+                    assert(str, "internal tokenizer error")
+                    -- replace possible %% chars
+                    str = str:gsub("%%%%", "%%%%<nowiki>%%%%</nowiki>%%%%")
+                    if tp == tokp.MARKUP_MONOSPACE then
+                        self:write_raw("%%''" .. str .. "''%%")
+                    else
+                        self:write_raw(str)
+                    end
                 end
-            else
-                self:write_raw(c)
-                c = f()
             end
         end
         self:write_raw("%%")
-        return self
-    end,
-
-    write_par = function(self, str)
-        local notetypes = M.has_feature("notes") and {
-            ["Note: "] = "<note>\n",
-            ["Warning: "] = "<note warning>\n",
-            ["Remark: "] = "<note tip>\n",
-            ["TODO: "] = "<note>\n**TODO:** "
-        } or {}
-        local tag
-        for k, v in pairs(notetypes) do
-            if str:match("^" .. k) then
-                tag = v
-                str = str:sub(#k + 1)
-                break
-            end
-        end
-        if tag then
-            self:write_raw(tag)
-            self:write_par_markup(str)
+        if hasnote then
             self:write_raw("\n</note>")
-        else
-            self:write_par_markup(str)
         end
         return self
     end,