def parse_config(files, commands):
    inner_globals = dict(SAFE_FUNCTIONS)
    inner_globals["__builtins__"] = {} # Disable any builtin function
    inner_globals["named_items"] = {} # Store the provided named items
    
    for file in files:
        text = file.read()
        root = ast.parse(text, filename=file.name)
        execute_code(root, file.name, inner_globals)
    
    for text in commands:
        root = ast.parse(text, filename="<execute>")
        execute_code(root, '--execute', inner_globals)
    
    # Make sure we have the necessary variables in the inner_globals dictionary
    for varname in REQUIRED:
        if varname not in inner_globals:
            raise ParseException("Missing the {!r} variable".format(varname))
    
    comparer = inner_globals["comparer"]
    groups = inner_globals["groups"]
    items = inner_globals["named_items"]
    
    if "total" in inner_globals:
        total = inner_globals["total"]
    else:
        total = len(groups)
    
    return Config(comparer, items, groups, total)

    def walk(filename, visitor):
        """Generate an AST for the given filename and walk over it using
        the given visitor instance.
        """

        filename = os.path.abspath(filename)

        try:
            tree = ast.parse(open(filename, 'r').read())
        except SyntaxError:
            if sys.version_info[0] < 3:
                e = sys.exc_info()[1]
                log.warn('SyntaxError while parsing file %s: %s' %
                         (filename, str(e)))
                return
            # We're probably in Python 3 and looking at a file intended for
            # Python 2.  Otherwise there's an unintended SyntaxError in the
            # file, so there are bigger problems anyways
            try:
                import lib2to3.refactor

                tool = StringRefactoringTool(
                    lib2to3.refactor.get_fixers_from_package('lib2to3.fixes'))
                tool.refactor_file(filename, write=True)
                tree = ast.parse(tool.refactored[filename])
            except ImportError:
                # Without 2to3 we can't do much more.
                # TODO: Issue a warning?
                return

        visitor.visit(tree)

def ParseCodeToTree(code):
  """Parse the given code to a lib2to3 pytree.

  Arguments:
    code: a string with the code to parse.

  Raises:
    SyntaxError if the code is invalid syntax.
    parse.ParseError if some other parsing failure.

  Returns:
    The root node of the parsed tree.
  """
  # This function is tiny, but the incantation for invoking the parser correctly
  # is sufficiently magical to be worth abstracting away.
  try:
    # Try to parse using a Python 3 grammar, which is more permissive (print and
    # exec are not keywords).
    parser_driver = driver.Driver(_GRAMMAR_FOR_PY3, convert=pytree.convert)
    tree = parser_driver.parse_string(code, debug=False)
  except parse.ParseError:
    # Now try to parse using a Python 2 grammar; If this fails, then
    # there's something else wrong with the code.
    try:
      parser_driver = driver.Driver(_GRAMMAR_FOR_PY2, convert=pytree.convert)
      tree = parser_driver.parse_string(code, debug=False)
    except parse.ParseError:
      # Raise a syntax error if the code is invalid python syntax.
      try:
        ast.parse(code)
      except SyntaxError as e:
        raise e
      else:
        raise
  return _WrapEndMarker(tree)

def scripts2rst(path, f):
    """ creates rst summary documentation for files in scripts folder
    which is not a package so cannot be imported """
       
    # get list of script files
    try:
        files = [name for name in os.listdir(path)
                    if not name.startswith("_")
                    and name.endswith(".py")]
    except:
        return
        
    f.write("**Scripts**\n\n")
    
    # loop over script files
    for name in files:
        sfile = os.path.join(path, name)
        try:
            try:
                source = ast.parse(open(sfile, "r", encoding="utf8").read())
            except:
                #py2 fails if encoding in string
                source = ast.parse(open(sfile, "r").read())
        except Exception as e:
            log.warning("Problem parsing %s\n%s"%(name, e))
        f.write(name+"\n")
        doc = i.cleandoc(ast.get_docstring(source)) or "."
        doc = py2decode(doc) #py2
        f.write(" "+doc.splitlines()[0]+"\n")
    f.write("\n")

def infer_ast(src):
    """Attempts to infer an abstract syntax tree from the provided value.
    
    - Python ast.AST instances are passed through.
    - Strings are parsed. A SyntaxError is raised if invalid.
    - Functions are sent through cypy.fn_get_source to get a source
      string, then parsed. If the source can't be a found, an exception is 
      raised by fn_get_source.
      
    .. WARNING:: Functions defined on the iPython command line do not have 
                 their source saved. A bug has been filed: 
                 
                 http://github.com/ipython/ipython/issues/issue/120
    """
    if isinstance(src, _ast.AST):
        return src
    elif isinstance(src, basestring):
        return _ast.parse(src)
    else:
        # if a function instance is passed in, it's source is found
        # and parsed. note that finding source can be precarious for
        # functions defined on the command line. If you get an error
        # you'll have to use strings instead of regular function
        # definitions
        src = cypy.fn_get_source(src)
        return _ast.parse(src)

 def update(self, name, script, template):
     self.name = name
     try:
         ast.parse(script)
         self.script = script
     except SyntaxError, e:
         raise UserException(e)

    def test_free_vars(self):
        stmt = ast.parse("foo", "", "single")
        with self.assertRaises(ValueError):
            freevars(stmt, [])

        suite = ast.parse("foo; bar();", "exec")
        with self.assertRaises(ValueError):
            freevars(suite, [])

        def freevars_(source, env=[]):
            return freevars(ast.parse(source, "", "eval"), env)

        self.assertEqual(freevars_("1"), [])
        self.assertEqual(freevars_("..."), [])
        self.assertEqual(freevars_("a"), ["a"])
        self.assertEqual(freevars_("a", ["a"]), [])
        self.assertEqual(freevars_("f(1)"), ["f"])
        self.assertEqual(freevars_("f(x)"), ["f", "x"])
        self.assertEqual(freevars_("f(x)", ["f"]), ["x"])
        self.assertEqual(freevars_("a + 1"), ["a"])
        self.assertEqual(freevars_("a + b"), ["a", "b"])
        self.assertEqual(freevars_("a + b", ["a", "b"]), [])
        self.assertEqual(freevars_("a[b]"), ["a", "b"])
        self.assertEqual(freevars_("a[b]", ["a", "b"]), [])
        self.assertEqual(freevars_("f(x, *a)", ["f"]), ["x", "a"])
        self.assertEqual(freevars_("f(x, *a, y=1)", ["f"]), ["x", "a"])
        self.assertEqual(freevars_("f(x, *a, y=1, **k)", ["f"]),
                         ["x", "a", "k"])
        if sys.version_info >= (3, 5):
            self.assertEqual(freevars_("f(*a, *b, k=c, **d, **e)", ["f"]),
                             ["a", "b", "c", "d", "e"])

        self.assertEqual(freevars_("True"), [])
        self.assertEqual(freevars_("'True'"), [])
        self.assertEqual(freevars_("None"), [])
        self.assertEqual(freevars_("b'None'"), [])

        self.assertEqual(freevars_("a < b"), ["a", "b"])
        self.assertEqual(freevars_("a < b <= c"), ["a", "b", "c"])
        self.assertEqual(freevars_("1 < a <= 3"), ["a"])

        self.assertEqual(freevars_("{}"), [])
        self.assertEqual(freevars_("[]"), [])
        self.assertEqual(freevars_("()"), [])
        self.assertEqual(freevars_("[a, 1]"), ["a"])
        self.assertEqual(freevars_("{a: b}"), ["a", "b"])
        self.assertEqual(freevars_("{a, b}"), ["a", "b"])
        self.assertEqual(freevars_("0 if abs(a) < 0.1 else b", ["abs"]),
                         ["a", "b"])
        self.assertEqual(freevars_("lambda a: b + 1"), ["b"])
        self.assertEqual(freevars_("lambda a: b + 1", ["b"]), [])
        self.assertEqual(freevars_("lambda a: a + 1"), [])
        self.assertEqual(freevars_("(lambda a: a + 1)(a)"), ["a"])
        self.assertEqual(freevars_("lambda a, *arg: arg + (a,)"), [])
        self.assertEqual(freevars_("lambda a, *arg, **kwargs: arg + (a,)"), [])

        self.assertEqual(freevars_("[a for a in b]"), ["b"])
        self.assertEqual(freevars_("[1 + a for c in b if c]"), ["a", "b"])
        self.assertEqual(freevars_("{a for _ in [] if b}"), ["a", "b"])
        self.assertEqual(freevars_("{a for _ in [] if b}", ["a", "b"]), [])

 def runcode(self, the_code, source, filename='<input>'):
     # code taken from InteractiveInterpreter.runsource in code.py
     try:
         tree = ast.parse(source)
         try:
             expr = ast.parse(source, mode='eval')
         except:
             expr = None
         #todo get this to work for multiple expr's, not just 1:
         if expr and len(tree.body) == 1:
             # _ = expr_value
             tree.body[0] = ast_wrap_in_assn('_', tree.body[0])
             # print _
             underscore = _ast.Name(id="_", ctx=_ast.Load())
             print_node = ast_print_node([_ast.Str(s=' '*50), underscore])
             tree.body.append(print_node)
             # play_whatever
                 #todo doesn't work for generators yet
             play_whatever_node = ast_call_node('music.play_whatever', '_', show_notes=SHOW_NOTES)
             tree.body.append(play_whatever_node)
         #print ast.dump(tree)
         code_obj = compile(tree, '<input>', 'exec')
         exec code_obj in self.locals
     except SystemExit:
         raise
     except:
         self.showtraceback()
     else:
         if code.softspace(sys.stdout, 0):
             print

def python_code(tokenIterator, envName, endToken):
    """
    Given an iterator of tokens, and the name of current environment, iterates through to the end of
    the environment, and returns the text in the environment as python code. This function only
    works for environments that are supposed to contain Python code.
    """
    count = 0
    pythonLine = ''
    pythonCode = []
    #parseNodes = []
    startLine = 0
    for token, lineNum in tokenIterator:
        if not startLine:
            #The first token is the endline at the end of \begin{envName}, so the environment starts on the same line as the first token in tokenIterator
            startLine = lineNum
        if endToken == token:
            pythonCode = [line for line in pythonCode if line.strip()]
            #We use this to figure out what the top level indent is, and strip that away so that Python can parse the code properly.
            topLevelIndent = 1 if pythonCode[0][0] == '\t' else compute_indent(pythonCode[0])
            try:
                ast.parse(''.join(line[topLevelIndent:] for line in  pythonCode))
            except SyntaxError, e:
                raise transExceptions.TranslationError(' '.join([parseTree.color("Error:", parseTree.bcolors.RED), 'Error in Python code found in', parseTree.color(envName,parseTree.bcolors.YELLOW),  
                    'environment. Environment start line:', parseTree.color(str(startLine), parseTree.bcolors.GREEN),  'Python error:\n\n', str(e)]))
            else:
                return (startLine, pythonCode)
        else:
            if token.strip(' \t') == '\n':
                pythonCode.append(pythonLine + token)
                pythonLine = ''
            else:
                pythonLine += token

    def warn_about_none_ast(self, node, module_path, lineno):
        """
        Returns an AST issuing a warning if the value of node is `None`.
        This is used to warn the user when asserting a function that asserts
        internally already.
        See issue #3191 for more details.
        """

        # Using parse because it is different between py2 and py3.
        AST_NONE = ast.parse("None").body[0].value
        val_is_none = ast.Compare(node, [ast.Is()], [AST_NONE])
        send_warning = ast.parse(
            """
from _pytest.warning_types import PytestWarning
from warnings import warn_explicit
warn_explicit(
    PytestWarning('asserting the value None, please use "assert is None"'),
    category=None,
    filename={filename!r},
    lineno={lineno},
)
            """.format(
                filename=module_path.strpath, lineno=lineno
            )
        ).body
        return ast.If(val_is_none, send_warning, [])

    def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

        node = ast.parse('async def foo():\n  """spam\n  ham"""')
        self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham')

	def get_changes(self):
		if not self.text_before: return
		tree_before = ast.parse(self.text_before)
		tree = ast.parse(self.text)
		if ast.dump(tree)==ast.dump(tree_before): print('status','no changes to the script')
		else: 
			print('status','executing changes to %s'%self.file)
			# identify changed nodes in the tree and execute
			# note that this feature reruns any changed child of the script parent
			#! track line numbers are report to the user?
			tree_before,tree = [[self.CodeChunk(i,index=ii) for ii,i in 
				enumerate(ast.iter_child_nodes(ast.parse(t)))]
				for t in [self.text_before,self.text]]
			intersect = set.intersection(set(tree),set(tree_before))
			novel = list(set.difference(set(tree),intersect))
			novel_linenos = set([i.this.lineno for i in novel])
			class CodeSurgery(ast.NodeTransformer):
				def visit(self, node):
					if hasattr(node,'lineno') and node.lineno not in novel_linenos: 
						return ast.parse('last_lineno = %d'%node.lineno).body[0]
					else: return ast.NodeTransformer.generic_visit(self,node)
			code_ready = ast.fix_missing_locations(CodeSurgery().visit(ast.parse(self.text)))
			# run the remainder
			out = self.namespace
			#! exec to eval for python <2.7.15
			eval(compile(code_ready,filename='<ast>',mode='exec'),out,out)

    def OnApply(self, event):
        """Event handler for Apply button"""

        # See if we have valid python
        try:
            ast.parse(self.macros)
        except:
            # Grab the traceback and print it for the user
            s = StringIO()
            e = exc_info()
            # usr_tb will more than likely be none because ast throws
            #   SytnaxErrorsas occurring outside of the current
            #   execution frame
            usr_tb = get_user_codeframe(e[2]) or None
            print_exception(e[0], e[1], usr_tb, None, s)
            post_command_event(self.parent, self.MacroErrorMsg,
                               err=s.getvalue())
            success = False
        else:
            self.result_ctrl.SetValue('')
            post_command_event(self.parent, self.MacroReplaceMsg,
                               macros=self.macros)
            post_command_event(self.parent, self.MacroExecuteMsg)
            success = True

        event.Skip()
        return success

def compare_strings(a,b):
    t1 = ast.parse(a)
    t2 = ast.parse(b)
    comp = Compare()
    val = comp.compare(t1,t2)
    d = comp.d
    return val

def _analyzeGens(top, absnames):
    genlist = []
    for g in top:
        if isinstance(g, _UserCode):
            tree = g
        elif isinstance(g, (_AlwaysComb, _AlwaysSeq, _Always)):
            f = g.func
            s = inspect.getsource(f)
            s = _dedent(s)
            tree = ast.parse(s)
            #print ast.dump(tree)
            tree.sourcefile  = inspect.getsourcefile(f)
            tree.lineoffset = inspect.getsourcelines(f)[1]-1
            tree.symdict = f.func_globals.copy()
            tree.callstack = []
            # handle free variables
            tree.nonlocaldict = {}
            if f.func_code.co_freevars:
                for n, c in zip(f.func_code.co_freevars, f.func_closure):
                    obj = _cell_deref(c)
                    if isinstance(g, _AlwaysComb):
                        if not ( isinstance(obj, (int, long, EnumType,_Signal)) or \
                                 _isMem(obj) or _isTupleOfInts(obj)
                               ):
                            info =  "File %s, line %s: " % (tree.sourcefile, tree.lineoffset)
                            print type(obj)
                            raise ConversionError(_error.UnsupportedType, n, info)
                    tree.symdict[n] = obj
                    # currently, only intbv as automatic nonlocals (until Python 3.0)
                    if isinstance(obj, intbv):
                        tree.nonlocaldict[n] = obj
            tree.name = absnames.get(id(g), str(_Label("BLOCK"))).upper()
            v = _FirstPassVisitor(tree)
            v.visit(tree)
            if isinstance(g, _AlwaysComb):
                v = _AnalyzeAlwaysCombVisitor(tree, g.senslist)
            elif isinstance(g, _AlwaysSeq):
                v = _AnalyzeAlwaysSeqVisitor(tree, g.senslist, g.reset, g.sigregs, g.varregs)
            else:
                v = _AnalyzeAlwaysDecoVisitor(tree, g.senslist)
            v.visit(tree)
        else: # @instance
            f = g.gen.gi_frame
            s = inspect.getsource(f)
            s = _dedent(s)
            tree = ast.parse(s)
            # print ast.dump(tree)
            tree.sourcefile = inspect.getsourcefile(f)
            tree.lineoffset = inspect.getsourcelines(f)[1]-1
            tree.symdict = f.f_globals.copy()
            tree.symdict.update(f.f_locals)
            tree.nonlocaldict = {}
            tree.callstack = []
            tree.name = absnames.get(id(g), str(_Label("BLOCK"))).upper()
            v = _FirstPassVisitor(tree)
            v.visit(tree)
            v = _AnalyzeBlockVisitor(tree)
            v.visit(tree)
        genlist.append(tree)
    return genlist

    def test_generate_files(self):
        """Test generate_files returns a tuple."""
        self.prototype.set_jinja_env(self.api_version)
        details = self.prototype.generate_files()
        self.assertIsInstance(details, list)
        # namedtuples in tuple
        for file_details in details:
            self.assertIsInstance(file_details, tuple)
            self.assertIsInstance(file_details.filename, basestring)
            self.assertIsInstance(file_details.filecontent, basestring)

            name, contents = file_details
            if name.endswith(".py"):
                # We have a "coding utf-8" line in there, we need to encode
                contents = contents.encode("utf-8")
                ast.parse(contents)
                if pep8:
                    checker = pep8.Checker(
                        name,
                        contents.splitlines(True))
                    res = checker.check_all()
                    self.assertFalse(
                        res,
                        "Python file {0} has pep8 errors:\n"
                        "{1}\n{2}".format(name, checker.report.messages,
                                          repr(contents))
                    )

            elif name.endswith(".xml"):
                # TODO validate valid odoo xml
                lxml.etree.fromstring(contents)

def veloce_generate_js(filepath, requirejs=False, root_path=None, output=None, deep=None):
    dirname = os.path.abspath(os.path.dirname(filepath))
    if not root_path:
        root_path = dirname
    basename = os.path.basename(filepath)
    output_name = os.path.join(dirname, basename + '.js')
    if not output:
        print('Generating {}'.format(output_name))
    # generate js
    with open(os.path.join(dirname, basename)) as f:
        input = parse(f.read())
    tree = parse(input)
    python_core = Veloce()
    python_core.visit(tree)
    script = python_core.writer.value()
    if requirejs:
        out = 'define(function(require) {\n'
        out += script
        if isinstance(python_core.__all__, str):
            out += '\nreturn {};\n'.format(python_core.__all__)
        elif python_core.__all__:
            public = '{{{}}}'.format(', '.join(map(lambda x: '{}: {}'.format(x[0], x[1]), zip(python_core.__all__, python_core.__all__))))
            out += '\nreturn {};\n'.format(public)
        else:
            raise Exception('__all__ is not defined!')
        out += '\n})\n'
        script = out
    if deep:
        for dependency in python_core.dependencies:
            if dependency.startswith('.'):
                generate_js(os.path.join(dirname, dependency + '.py'), requirejs, root_path, output, deep)
            else:
                generate_js(os.path.join(root_path, dependency[1:] + '.py'), requirejs, root_path, output, deep)
    output.write(script)

def pythonium_generate_js(filepath, requirejs=False, root_path=None, output=None, deep=None):
    dirname = os.path.abspath(os.path.dirname(filepath))
    if not root_path:
        root_path = dirname
    basename = os.path.basename(filepath)
    output_name = os.path.join(dirname, basename + '.js')
    if not output:
        print('Generating {}'.format(output_name))
    # generate js
    with open(os.path.join(dirname, basename)) as f:
        input = parse(f.read())
    tree = parse(input)
    pythonium = Pythonium()
    pythonium.visit(tree)
    script = pythonium.writer.value()
    if requirejs:
        out = 'define(function(require) {\n'
        out += script
        all = map(lambda x: "'{}': {}".format(x, x), pythonium.__all__)
        all = '{{{}}}'.format(', '.join(all))
        out += 'return {}'.format(all)
        out += '\n})\n'
        script = out
    if deep:
        for dependency in python_core.dependencies:
            if dependency.startswith('.'):
                generate_js(os.path.join(dirname, dependency + '.py'), requirejs, root_path, output, deep)
            else:
                generate_js(os.path.join(root_path, dependency[1:] + '.py'), requirejs, root_path, output, deep)
    output.write(script)

  def __init__(self, ast_node=None, source=None, path=None, filename=None):
    if filename:
      with open(filename, 'r') as file_obj:
        source = file_obj.read()
      ast_node = ast.parse(source)
      self._filename = filename
      self._path = path

    elif path:
      source = inspect.getmodule()
      ast_node = ast.parse(self._source)
      self._filename = inspect.getsourcefile(Import(path=path)._import)
      self._path = path

    else:
      self._path = None
      self._filename = None

    if ast_node and not isinstance(ast_node, ast.Module):
      raise TypeError('Expected an ast.Module object')

    super(Module, self).__init__(ast_node=ast_node, source=source)

    map( # set metadata properties on the object
      lambda x: setattr(self, x, self.get_var('__' + x + '__')),
      METADATA
    )
    self._type = 'Module'

    def __init__(self, network, name, source, target, parameters):
        Connector.__init__(self, network, name, source, target, parameters)

        # lets load up the weight ModularConnectorFunction's
        self.weight_functions = {}
        self.delay_functions = {}
        self.simulator_time_step = self.sim.get_time_step()
        # lets determine the list of variables in weight expressions
        v = ExpVisitor()
        v.visit(ast.parse(self.parameters.weight_expression))
        self.weight_function_names = v.names
        # lets determine the list of variables in delay expressions
        v = ExpVisitor()
        v.visit(ast.parse(self.parameters.delay_expression))
        self.delay_function_names = v.names

        for k in self.weight_function_names:
            self.weight_functions[k] = load_component(self.parameters.weight_functions[k].component)(
                self.source, self.target, self.parameters.weight_functions[k].params
            )
            assert isinstance(self.weight_functions[k], ModularConnectorFunction)

        for k in self.delay_function_names:
            self.delay_functions[k] = load_component(self.parameters.delay_functions[k].component)(
                self.source, self.target, self.parameters.delay_functions[k].params
            )