def _parse_docstring(self):
        """Parses the test docstring extracting expected values.

        If the expected tags is not spelled right they will not be parsed.
        """
        if self.docstring is None:
            return

        # Create the contexts
        tags, unexpected_tags, _ = self._parse_tags(
            ast.get_docstring(self.module_def))
        class_tags, class_unexpected_tags, _ = self._parse_tags(
            ast.get_docstring(self.parent_class_def))
        function_tags, function_unexpected_tags, self.skipped_lines = (
            self._parse_tags(self.docstring))

        # Update context dictionaries
        tags.update(class_tags)
        tags.update(function_tags)
        unexpected_tags.update(class_unexpected_tags)
        unexpected_tags.update(function_unexpected_tags)

        for tag, value in tags.items():
            if tag == 'bz':
                tag = 'bugs'
            if tag == 'assert':
                tag = 'assertion'
            if tag == 'type':
                tag = 'test_type'
            setattr(self, tag, value)
        self.unexpected_tags = unexpected_tags

        # Always use the first line of docstring as test case name
        if self.test is None:
            self.test = self.docstring.strip().split('\n')[0]

	def extract_info(self):
		for node in self.module.body: 
			if isinstance(node, ast.ClassDef):
				yield {
					"name": node.name,
					"lineno": node.lineno,
					"docstring": ast.get_docstring(node),
					"type": 'class',
					}
				for sub_node in node.body:
					if isinstance(sub_node, ast.FunctionDef):
						yield {
							"name": sub_node.name,
							"lineno": sub_node.lineno,
							"docstring": ast.get_docstring(sub_node),
							"type": 'attribute',
							"args": [arg.id for arg in sub_node.args.args],
							"header": ''
							}

			elif isinstance(node, ast.FunctionDef):
				yield {
					"name": node.name,
					"lineno": node.lineno,
					"docstring": ast.get_docstring(node),
					"type": 'function',
					"args": [arg.id for arg in node.args.args],
					}

    def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

        node = ast.parse('async def foo():\n  """spam\n  ham"""')
        self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham')

 def __init__(self, function_def, parent_class=None, testmodule=None):
     """Wrap a ``ast.FunctionDef`` instance used to extract information."""
     self.docstring = ast.get_docstring(function_def)
     self.function_def = function_def
     self.name = function_def.name
     if parent_class:
         self.parent_class = parent_class.name
         self.parent_class_def = parent_class
         self.class_docstring = ast.get_docstring(self.parent_class_def)
     else:
         self.parent_class = None
         self.parent_class_def = None
         self.class_docstring = None
     self.testmodule = testmodule.path
     self.module_def = testmodule
     self.module_docstring = ast.get_docstring(self.module_def)
     self.pkginit = os.path.join(
         os.path.dirname(self.testmodule), '__init__.py')
     if os.path.exists(self.pkginit):
         self.pkginit_def = ast.parse(''.join(open(self.pkginit)))
         self.pkginit_docstring = ast.get_docstring(self.pkginit_def)
     else:
         self.pkginit_def = None
         self.pkginit_docstring = None
     self.tokens = {}
     self.invalid_tokens = {}
     self._rst_parser_messages = []
     self.parser = DocstringParser(
         SETTINGS.get('tokens'),
         SETTINGS.get('minimum_tokens'),
     )
     self._parse_docstring()

 def collect_docstrings(self):
     """Collect list of paths + classes + methods/functions, and docstrings.
     
     This function assumes that there are no classes or methods/functions
     nested within others.
     """
     self.node_sequence = []
     # Module-level docstrings.
     self.node_sequence.append([self.path, ast.get_docstring(self.module)])
     # Class-level doc-strings
     # Function-level doc-strings
     for class_def in self.class_defs:
         for node in class_def.body:
             if isinstance(node, ast.ClassDef):
                 self.node_sequence.append(
                         [self.path + '.' + node.name,
                          ast.get_docstring(node)]
                         )
             elif isinstance(node, ast.FunctionDef):
                 if self.tests_only and node.name[:5] != 'test_':
                     continue
                 self.node_sequence.append(
                         [self.path + '.' + class_def.name + '.' + node.name,
                          ast.get_docstring(node)]
                         )
     for func_def in self.func_defs:
         if isinstance(func_def, ast.FunctionDef):
             if self.tests_only and func_def.name[:5] != 'test_':
                 continue
             self.node_sequence.append(
                     [self.path + '.' + func_def.name,
                      ast.get_docstring(func_def)]
                     )

def parse_module(file_path):
    module = ast.parse(file_path.read_text())
    tmpl_str = HEADER.format(file_path.name.rsplit('.',1)[0])
    cls_defs = [node for node in module.body if isinstance(node, ast.ClassDef)]
    mod_func_defs = [node for node in module.body if isinstance(node, ast.FunctionDef)]
    for cls_def in cls_defs:
        cls_name = cls_def.name
        cls_bases = ','.join([parse(each) for each in cls_def.bases])
        tmpl_str += f'== {{{{class {cls_name}{":" + cls_bases if cls_bases else ""}}}}}\n\n'
        method_str = None
        for fn_def in (fn_def for fn_def in cls_def.body if isinstance(fn_def, ast.FunctionDef)):
            if fn_def.name == '__init__':
                tmpl_str += "=== Arguments\n" + parse_args(fn_def.args) + "\n\n"
            else:
                if not method_str:
                    method_str = '=== Methods\n\n'
                doc_str = ast.get_docstring(fn_def)
                method_str += f'{{{{method {fn_def.name},{doc_str if doc_str else ""}}}}}\n\n'
        tmpl_str += method_str if method_str else ''
    method_str = None
    for fn_def in mod_func_defs:
        if not method_str:
            method_str = '== Module Functions\n\n'
        doc_str = ast.get_docstring(fn_def)
        method_str += f'{{{{method {fn_def.name},{doc_str if doc_str else ""}}}}}\n\n'
    tmpl_str += method_str if method_str else ''
    return tmpl_str

 def get_doc(self, module):
     for node in module.body:
         if isinstance(node, ast.ClassDef):
             yield ast.get_docstring(node)
             for sub_node in node.body:
                 if isinstance(sub_node, ast.FunctionDef):
                     yield ast.get_docstring(sub_node)

def _getMarvinTestDocStrings(classname, testnames, marvinCodePath):
    pathToClass = os.path.join(marvinCodePath, *classname.split('.')[1:-1])+'.py'
    astData = ast.parse(open(pathToClass).read())
    classElement = filter(lambda x:isinstance(x, ast.ClassDef) and x.name == classname.split('.')[-1], astData.body)[0]
    classDocString = ast.get_docstring(classElement)
    classDocString = classDocString and classDocString.rstrip() or ''
    testMethodElements = filter(lambda x:isinstance(x, ast.FunctionDef) and x.name in testnames, classElement.body)
    testMethodDocStrings = []
    for testMethod in testMethodElements:
        docStr = ast.get_docstring(testMethod)
        docStr = docStr and docStr.rstrip() or ''
        testMethodDocStrings.append((testMethod.name, docStr))
    return (classDocString, testMethodDocStrings)

    def visit_FunctionDef(self, node, **kwargs):
        """
        Handles function definitions within code.

        Process a function's docstring, keeping well aware of the function's
        context and whether or not it's part of an interface definition.
        """
        if self.options.debug:
            stderr.write("# Function {0.name}{1}".format(node, linesep))
        # Push either 'interface' or 'class' onto our containing nodes
        # hierarchy so we can keep track of context.  This will let us tell
        # if a function is nested within another function or even if a class
        # is nested within a function.
        containingNodes = kwargs.get('containingNodes', []) or []
        containingNodes.append((node.name, 'function'))
        if self.options.topLevelNamespace:
            fullPathNamespace = self._getFullPathName(containingNodes)
            contextTag = '.'.join(pathTuple[0] for pathTuple in fullPathNamespace)
            modifiedContextTag = self._processMembers(node, contextTag)
            tail = '@namespace {0}'.format(modifiedContextTag)
        else:
            tail = self._processMembers(node, '')
        if get_docstring(node):
            self._processDocstring(node, tail,
                                   containingNodes=containingNodes)
        # Visit any contained nodes.
        self.generic_visit(node, containingNodes=containingNodes)
        # Remove the item we pushed onto the containing nodes hierarchy.
        containingNodes.pop()

def _get_file_content(realpath):
    """Helper function to turn a file into HTML"""

    result = """<h2>Module: %s</h2>
       <div class="alert alert-info">
       <div style = "details">path: %s</div>
       <div style = "details">created: %s, last modified: %s</div></div>""" %(
          os.path.basename(realpath),
          urlparse.unquote(_convert_path_to_url(realpath)),
          time.ctime(os.path.getmtime(realpath)),
          time.ctime(os.path.getctime(realpath))
    )
    with open(realpath) as sourcefile:
        code = sourcefile.readlines()
        abstract_syntax_tree = ast.parse(''.join(code))
        description = ast.get_docstring(abstract_syntax_tree)
        if description:
            result += "<h3>Summary:</h3> <div style = 'summary'>%s</div>" % _convert_str_to_html(description)
        visitor = DocVisitor()
        visitor.visit(abstract_syntax_tree)
        parsed_code = visitor.get_doc()
        entries = [ parsed_code[key] for key in sorted(parsed_code.keys())]
        for entry in entries:
            begin, end = entry.get("lines")
            result += '<hr /><div><pre>' + "".join(code[begin:end]).rstrip().rstrip(":") +"</pre>"
            result += _convert_str_to_html(entry.get("description"))+"</div>"

    return result

def header():
   '''
   displays current module docstring
   '''
   f=inspect.stack()[1][1]
   m=ast.parse(''.join(open(f)))
   print "\n%s" % ast.get_docstring(m)

    def parse_tree (self, name, node):
        
        """
        Recursive function that explores the python parse tree and
        constructs our TreeNode datastructure to represent a document. 

        Takes the root node of the tree provided by the std ast module and
        the file name.
        """
        
        logging.debug('Exploring node %s of %s' % (node, name))

        tree_node = ParserNode(
            type = type_lookup[type(node)],
            name = node.name if 'name' in node.__dict__ else \
                   name.split('.')[0],
            docstring = ast.get_docstring(node),
            children = [],
            terms = []
        )

        for child in node.body:

            # We only want to look for modules, functions and classes as
            # these are the only items that have doc strings and therefore
            # terms at the moment. Later on, we'll want to use imports and
            # function names and other elements of the parse tree.
            if type(child) not in [ast.Module, ast.FunctionDef, ast.ClassDef]:
                continue

            tree_node.children.append(self.parse_tree(name, child))

        return tree_node

def scripts2rst(path, f):
    """ creates rst summary documentation for files in scripts folder
    which is not a package so cannot be imported """
       
    # get list of script files
    try:
        files = [name for name in os.listdir(path)
                    if not name.startswith("_")
                    and name.endswith(".py")]
    except:
        return
        
    f.write("**Scripts**\n\n")
    
    # loop over script files
    for name in files:
        sfile = os.path.join(path, name)
        try:
            try:
                source = ast.parse(open(sfile, "r", encoding="utf8").read())
            except:
                #py2 fails if encoding in string
                source = ast.parse(open(sfile, "r").read())
        except Exception as e:
            log.warning("Problem parsing %s\n%s"%(name, e))
        f.write(name+"\n")
        doc = i.cleandoc(ast.get_docstring(source)) or "."
        doc = py2decode(doc) #py2
        f.write(" "+doc.splitlines()[0]+"\n")
    f.write("\n")

 def visit_FunctionDef(self, tree):
     # self.visit() returns something of the form
     # ('lambda x, y, z=5, *args: ', ['x', 'y', 'z', 'args'])
     args, arg_names = self.visit(tree.args)
     decoration = T('{}')
     for decorator in tree.decorator_list:
         decoration = decoration.format(T('{}({})').format(self.visit(decorator), T('{}')))
     ns = self.next_child()
     body = ns.many_to_one(tree.body).format(pre_return='', post_return='')
     if arg_names:
         body = assignment_component(body,
             T(', ').join(ns.var(name) for name in arg_names),
             T(', ').join(arg_names))
     body = self.close(ns, '{}', body)
     function_code = args + body
     doc = ast.get_docstring(tree, clean=False)
     if tree.decorator_list:
         return assignment_component(
             T('{after}'),
             self.store_var(tree.name),
             decoration.format(assignment_component(
                 '__func',
                 '__func, __func.__name__' + ('' if doc is None else ', __func.__doc__'),
                 T('{}, {!r}' + ('' if doc is None else ', {!r}')).format(
                     function_code, tree.name, doc))))
     else:
         return assignment_component(
             T('{after}'),
             T('{}, {}.__name__' + ('' if doc is None else ', {}.__doc__')).format(
                 self.store_var(tree.name), self.var(tree.name), self.var(tree.name)),
             T('{}, {!r}' + ('' if doc is None else ', {!r}')).format(
                 function_code, tree.name, doc))

 def shortdesc():
     filename = os.path.join(PACKAGE, '__init__.py')
     data = Setup.read(filename)
     node = ast.parse(data, filename)
     docstring = ast.get_docstring(node)
     desc = docstring.strip().split('\n\n', 1)[0]
     return desc.replace('\n', ' ')

def _parse_class(symbol, with_docstrings):
    docstring = {}
    attr = {}
    func = {}
    clazz = {}
    name = symbol.name + '('
    name += ', '.join([
        analyzer.expand_attribute(base) for base in symbol.bases])
    name += ')'
    for sym in symbol.body:
        if sym.__class__ is ast.Assign:
            result = _parse_assign(sym)
            attr.update(result[0])
            attr.update(result[1])
        elif sym.__class__ is ast.FunctionDef:
            result = _parse_function(sym, with_docstrings)
            attr.update(result['attrs'])
            if with_docstrings:
                docstring.update(result['docstring'])
            func[result['name']] = (result['lineno'], result['functions'])
        elif sym.__class__ is ast.ClassDef:
            result = _parse_class(sym, with_docstrings)
            clazz[result['name']] = (result['lineno'],
                {'attributes': result['attributes'],
                'functions': result['functions']})
            docstring.update(result['docstring'])
    if with_docstrings:
        docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True)

    lineno = symbol.lineno
    for decorator in symbol.decorator_list:
        lineno += 1

    return {'name': name, 'attributes': attr, 'functions': func,
        'lineno': lineno, 'docstring': docstring, 'classes': clazz}

    def __init__(self, node):
        name = node.name
        args = []
        if node.args.args is not None:
            for argument in node.args.args:
                args.append(argument.arg)
            for default, num_arg in zip(
                    reversed(node.args.defaults), reversed(range(len(args)))):
                if type(default) is _ast.Str:
                    args[num_arg] += ' = "' + default.s + '"'
                elif type(default) is _ast.NameConstant:
                    args[num_arg] += " = " + str(default.value)
                elif type(default) is _ast.Num:
                    args[num_arg] += " = " + str(default.n)
                elif type(default) is _ast.Name:
                    args[num_arg] += " = " + str(default.id)

        if node.args.vararg is not None:
            args.append("*" + node.args.vararg.arg)

        if node.args.kwarg is not None:
            args.append("**" + node.args.kwarg.arg)

        doc = ast.get_docstring(node)
        super(Function, self).__init__("Function", name, args, doc)

 def _extract_default_fits_file(self, path):
     with open(path, 'r') as rfile:
         m = ast.parse(rfile.read())
         docstr = ast.get_docstring(m)
         yd = yaml.load(docstr)
         if yd:
             return yd.get('default_fits', None)

def _parse_function(symbol, with_docstrings):
    docstring = {}
    attrs = {}
    func = {'functions': {}}

    func_name = symbol.name + '('
    #We store the arguments to compare with default backwards
    defaults = []
    for value in symbol.args.defaults:
        #TODO: In some cases we can have something like: a=os.path
        defaults.append(value)
    arguments = []
    for arg in reversed(symbol.args.args):
        if arg.__class__ is not _ast.Name or arg.id == 'self':
            continue
        argument = arg.id
        if defaults:
            value = defaults.pop()
            arg_default = _map_type.get(value.__class__, None)
            if arg_default is None:
                if value.__class__ is _ast.Attribute:
                    arg_default = analyzer.expand_attribute(value)
                elif value.__class__ is _ast.Name:
                    arg_default = value.id
                else:
                    arg_default = 'object'
            argument += '=' + arg_default
        arguments.append(argument)
    func_name += ', '.join(reversed(arguments))
    if symbol.args.vararg is not None:
        if not func_name.endswith('('):
            func_name += ', '
        func_name += '*' + symbol.args.vararg
    if symbol.args.kwarg is not None:
        if not func_name.endswith('('):
            func_name += ', '
        func_name += '**' + symbol.args.kwarg
    func_name += ')'

    for sym in symbol.body:
        if sym.__class__ is ast.Assign:
            result = _parse_assign(sym)
            attrs.update(result[1])
        elif sym.__class__ is ast.FunctionDef:
            result = _parse_function(sym, with_docstrings)
            if with_docstrings:
                docstring.update(result['docstring'])
            func['functions'][result['name']] = {'lineno': result['lineno'],
                                                 'functions':
                                                 result['functions']}

    if with_docstrings:
        docstring[symbol.lineno] = ast.get_docstring(symbol, clean=True)

    lineno = symbol.lineno
    for decorator in symbol.decorator_list:
        lineno += 1

    return {'name': func_name, 'lineno': lineno,
            'attrs': attrs, 'docstring': docstring, 'functions': func}

 def _parse_py_file(py_file):
     tree = ast.parse("".join(open(py_file)))
     # noinspection PyArgumentEqualDefault
     docstring = (ast.get_docstring(tree, clean=True) or "").strip()
     functions = [node.name for node in tree.body if type(node) == ast.FunctionDef]
     classes = [node.name for node in tree.body if type(node) == ast.ClassDef]
     return docstring, functions, classes