non-method functions support for python indexer in gendocsets

- number of entries increased from 2084 to 3781

README.md

@@ -18,7 +18,7 @@ Zeal is a simple documentation browser inspired by [Dash](http://kapeli.com/dash
 Currently Zeal requires Qt 5.0. After compiling it you need to download docsets and put them in `$HOME/.local/share/zeal/docsets/` -- after creating the directory first. Currently there are docsets available from Qt 5 and Python 2.7.3, generated using scripts from `gendoctests` directory, but for convenience can be downloaded from Dropbox:
 
  * [Qt5.tar.bz2](https://www.dropbox.com/s/o8uy2iz0wivs04j/Qt5.tar.bz2) (55M)
- * [python-2.7.3-docs-html.tar.bz2](https://www.dropbox.com/s/weve3snkc9gpqlj/python-2.7.3-docs-html.tar.bz2) (4.3M)
+ * [python-2.7.3-docs-html.tar.bz2](https://www.dropbox.com/s/2xehy4d2umwg574/python-2.7.3-docs-html.tar.bz2) (4.3M)
 
 Do `tar -jxvf file.tar.bz2` in docsets directory to enable given docset.
 
@@ -29,6 +29,7 @@ Do `tar -jxvf file.tar.bz2` in docsets directory to enable given docset.
    1. Allow selecting subset of docsets to search in.
    2. Substring match in case current startswith matching doesn't return anything.
    3. Better sorting of results (currently they are grouped by docset, which is probably not a good idea)
+   4. Better docsets formatting (without headers, sidebars etc.)
 
 
 ## Contributions

gendocsets/python273/crawl.py

@@ -11,17 +11,21 @@ c.execute('CREATE TABLE things (id integer primary key, type text, name text, pa
 
 tree = parse('py-modindex.html')
 
+modules = {}
 modfiles = set()
 
 for tbl in tree.xpath('//table[@class="indextable modindextable"]'):
     for tr in tbl.findall('tr'):
         a = tr.findall('td')[1].find('a')
         if a is None: continue
-        c.execute('INSERT INTO things(type, name, path) values("module", ?, ?)', (a.find('tt').text, a.attrib['href']))
+        modname = a.find('tt').text
+        c.execute('INSERT INTO things(type, name, path) values("module", ?, ?)', (modname, a.attrib['href']))
+        modules[modname] = c.lastrowid
         modfiles.add(a.attrib['href'].split('#')[0])
 
+
 def parseClass(class_id, url, tree):
-    for dl in tree.xpath('dd/dl[@class="method"]'):
+    for dl in tree.xpath('dd/dl[@class="method" or @class="function"]'):
         url = fname
         if dl.xpath('dt/@id'):
             url += '#'+dl.xpath('dt/@id')[0]
@@ -32,14 +36,44 @@ def parseClass(class_id, url, tree):
 for fname in modfiles:
     tree = parse(fname)
     for cls in tree.xpath('//dl[@class="class"]'):
-        from lxml.etree import tostring
         header = cls.find('dt')
         url = fname
-        if 'id' in header.attrib:
-            url += '#' + header.attrib['id']
-        c.execute('INSERT INTO things(type, name, path) values("class", ?, ?)', 
-            (cls.xpath('dt/tt[@class="descname"]/text()')[0], url))
+        modname = cls.xpath('dt/tt[@class="descclassname"]/text()')[0][:-1]
+        # missing modules
+        if modname.startswith('email.mime.'): modname = 'email.mime'
+        if modname == 'multiprocessing.queues': modname = 'multiprocessing'
+        modid = modules[modname]
+        if 'id' not in header.attrib:
+            continue
+        url += '#' + header.attrib['id']
+        c.execute('INSERT INTO things(type, name, path, parent) values("class", ?, ?, ?)', 
+            (cls.xpath('dt/tt[@class="descname"]/text()')[0], url, modid))
         parseClass(c.lastrowid, fname, cls)
+    
+    # non-method functions:
+    for function in tree.xpath('//dl[@class="function"]'):
+        # ignore methods with class="function"
+        if function.getparent().getparent().attrib.get('class') == 'class': continue
+        # ctypes.html - callable (ugh.)
+        if function.getparent().getparent().getparent().getparent().attrib.get('class') == 'class': continue
+        header = function.find('dt')
+        url = fname
+        from lxml.etree import tostring
+        modname = function.xpath('dt/tt[@class="descclassname"]/text()')
+        if modname:
+            modname = modname[0][:-1]
+        else:
+            # ctypes prototype
+            continue
+        # missing modules
+        if modname == 'ctypes.util': modname = 'ctypes'
+        modid = modules[modname]
+        if 'id' not in header.attrib:
+            continue
+        url += '#' + header.attrib['id']
+        c.execute('INSERT INTO things(type, name, path, parent) values("function", ?, ?, ?)', 
+            (function.xpath('dt/tt[@class="descname"]/text()')[0], url, modid))
+
 
 std_classes = {}
 def get_std_class(name, href):
@@ -49,7 +83,6 @@ def get_std_class(name, href):
         std_classes[name] = c.lastrowid
     return std_classes[name]
 
-    
 
 tree = parse('library/stdtypes.html')
 for method in tree.xpath('//dl[@class="method"]'):
@@ -61,9 +94,6 @@ for method in tree.xpath('//dl[@class="method"]'):
         classname = classname[0][:-1]
     else:
         classname = method.getparent().getparent().find('dt').find('tt').text
-    print ('INSERT INTO things(type, name, path, parent) values("member", ?, ?, ?)',
-        (method.xpath('dt/tt[@class="descname"]/text()')[0], url,
-         get_std_class(classname, url)))
     c.execute('INSERT INTO things(type, name, path, parent) values("member", ?, ?, ?)',
         (method.xpath('dt/tt[@class="descname"]/text()')[0], url,
          get_std_class(classname, url)))