def start_mzbench_server():
    if 'MZBENCH_RSYNC' in os.environ:
        node_location_param = '{{node_rsync, "{0}"}},'.format(os.environ['MZBENCH_RSYNC'])
    elif 'MZBENCH_REPO' in os.environ:
        node_location_param = '{{node_git, "{0}"}},'.format(os.environ['MZBENCH_REPO'])
    else:
        node_location_param = ''

    with open(dirname + "/test_server.config", "w") as config:
        config.write('[{{mzbench_api, [{0}{{node_log_port, 0}}, {{node_management_port, 0}}]}}].'.format(node_location_param))

    with open('{0}/test_server.config'.format(dirname), 'r') as f:
        print(f.read())

    cmd('{0} start_server --config {1}/test_server.config'.format(mzbench_script, dirname))
    try:
        time.sleep(3) # give server some time to start
        yield
    except:
        print ''
        print '-------------------- >> begin server logs << ---------------------'
        logdir = os.path.join(mzbench_dir + 'server/_build/default/rel/mzbench_api/log')
        logfiles = [logfile for logfile in os.listdir(logdir)]
        logfile = sorted([os.path.join(logdir, l) for l in logfiles if l.startswith('erlang')], key=os.path.getmtime, reverse=True)[0]
        with open(logfile) as f:
            for line in f:
                print line.rstrip().replace('\\n', '\n')
        print '-------------------- >> end server logs   << ---------------------'
        print ''
        raise
    finally:
        cmd('{0} stop_server'.format(mzbench_script))

    def pack(self):
        emb_root = self.target_root
        if self.seed:
            emb_root = emb_root.pjoin(self.target_root)

        basedir = util.Path( os.path.dirname(self.tarpath) )
        util.mkdir(basedir)

        archive = tarfile.open(self.tarpath, 'w:bz2')
        archive.add(emb_root,
            arcname = '/',
            recursive = True
        )
        archive.close()

        curdir = os.path.realpath(os.curdir)
        os.chdir(emb_root)
        util.cmd('find ./ | cpio -H newc -o | gzip -c -9 > %s' % (self.cpiopath))
        os.chdir(curdir)

        if self.kernel:
            r = util.Path('/')
            if self.seed:
                r = self.target_root
            r = r.pjoin('/tmp/inhibitor/kernelbuild')

            kernel_link = r.pjoin('/boot/kernel')
            kernel_path = os.path.realpath( kernel_link )

            if os.path.lexists(self.kernlinkpath):
                os.unlink(self.kernlinkpath)

            shutil.copy2(kernel_path, basedir.pjoin( os.path.basename(kernel_path) ))
            os.symlink(os.path.basename(kernel_path), self.kernlinkpath)

    def target_merge_busybox(self):
        env = {}
        use = ''
        if 'USE' in self.env:
            use = self.env['USE']

        if len(self.package_list) == 0:
            use += ' static'
        use += ' make-symlinks'
        env.update(self.env)
        env['USE'] = use
        env['ROOT'] = self.target_root

        cmdline = '%s/inhibitor-run.sh run_emerge --newuse --nodeps sys-apps/busybox' \
                % self.env['INHIBITOR_SCRIPT_ROOT']

        if self.seed:
            util.chroot(
                path        = self.target_root,
                function    = util.cmd,
                fargs       = {'cmdline':cmdline, 'env':env},
                failuref    = self.chroot_failure
            )
            util.chroot(
                path        = self.target_root,
                function    = self.path_sync_callback,
                fargs       = {'src':'/bin/busybox', '_':None},
                failuref    = self.chroot_failure
            )

        else:
            util.cmd( cmdline, env = env )
            self.path_sync_callback('/bin/busybox', None)

	def restore(self, source, specificFileToRestore='', restoreTo=''):
		"""source: name of source group to restore
		specificFileToRestore: relative path of a specific file/directory to restore
		restoreTo: path string of restore location"""
		sourceGrp = self.sourceGroups[source]
		if sourceGrp is None: 
			raise Exception("Cannot restore: no such source group "+
							"associated with link - {0}".format(source))
		os.environ['PASSPHRASE'] = getpwd()
		options = []
		
		# duplicity currently doesn't support the 'include' 
		# and 'exclude' options when performing a restore
		# these options are what allow us to backup in a single command above.
		# instead, we use a series of multiple commands with 'file-to-restore' option
		# to perform the same role.
		for source in souceGrp.items:
			path = source.path
			# We must make it a relative path (i.e. remove leading slash)
			if path.startswith('/'):
				path = path[1:]
			cmdList = ["duplicity",
						"restore",
						"--file-to-restore='{0}'".format(path),
						self.getTarget().path,
						'/' if restoreTo is '' else restoreTo
						]
			cmd(cmdList)
	
		# XXX
		#if specificFileToRestore is not '':
		#	options.append('--file-to-restore="{0}"'.format(specificFileToRestore))
				
		del os.environ['PASSPHRASE']
		self.doneSomething()

    def sync(self):
        if os.path.exists(self.builddir):
            shutil.rmtree(self.builddir)
        elif os.path.islink(self.builddir):
            os.unlink(self.builddir)
        os.makedirs(self.builddir)

        exclude_cmd = ''
        if self.exclude:
            for i in self.exclude:
                exclude_cmd += " --exclude='%s'" % i

        if self.include:
            for pattern in self.include:
                paths = [self.src.cachedir.pjoin(pattern)]
                if '*' in pattern:
                    paths = glob.glob(self.src.cachedir.pjoin(pattern))

                for path in paths:
                    dest = path.replace(self.src.cachedir, self.builddir)
                    if not os.path.lexists( os.path.dirname(dest) ):
                        os.makedirs( os.path.dirname(dest) )
                    util.cmd('rsync -a %s %s/ %s/' % (
                        exclude_cmd,
                        path,
                        dest
                    ))
        else:
            util.cmd('rsync -a %s %s/ %s/' % (exclude_cmd, self.src.cachedir, self.builddir))

def check_migrations_test():
    mzbench_data_dir = tempfile.mkdtemp(prefix='mzbench_data_')

    with start_mzbench_server(custom_data_location=mzbench_data_dir):
        for i in range(5):
            run_successful_bench(scripts_dir + 'correct_script.erl')

    try:
        cmd(mzbench_dir + '/bin/migrate.py ' + mzbench_data_dir)
    finally:
        shutil.rmtree(mzbench_data_dir)

def rm_tabsint_plugin_dependencies(pluginDict):
    """ remove cordova dependencies part of plugins from config """
    try:
        for name, plugin in pluginDict.iteritems():
            if "cordova" in plugin:
                ut.cmd("npm run cordova --silent -- plugin rm " + plugin["cordova"]["package"], force=True)
    except:
        sys.exit(
            "[ERROR]: Error while trying to uninstall plugin: {0}. Try manually uninstalling using 'npm run cordova --silent -- plugin rm {1}'".format(
                plugin["package"], plugin["package"]
            )
        )

    def merge_packages(self):
        cmdline = '%s/inhibitor-run.sh run_emerge --newuse %s' % (
            self.env['INHIBITOR_SCRIPT_ROOT'], ' '.join(self.package_list))

        if self.seed:
            util.chroot(
                path        = self.target_root,
                function    = util.cmd,
                fargs       = {'cmdline':cmdline, 'env':self.env},
                failuref    = self.chroot_failure
            )
        else:
            util.cmd( cmdline, env = self.env )

def add_tabsint_plugin_dependencies(pluginDict):
    """ add dependent cordova plugins for tabsint plugins listed in config.json"""
    try:
        for name, plugin in pluginDict.iteritems():
            if "cordovaPlugins" in plugin:
                for p in plugin["cordovaPlugins"]:
                    ut.cmd("npm run cordova --silent -- plugin add {0}".format(p))
                    ut.log.info(
                        '[BUILD]: Successfully installed the cordova plugin "{0}" required by tabsint plugin "{1}"'.format(
                            p, name
                        )
                    )
    except Exception as e:
        rm_tabsint_plugins()
        sys.exit('[ERROR]: Error while trying to "cordovaPlugins" from with error: {0}'.format(str(e)))

    def unpack_seed(self):
        if not os.path.isdir(self.seed):
            if os.path.exists(self.seed):
                os.unlink(self.seed)
            seedfile = self.seed + ".tar.bz2"
            util.info("Unpacking %s" % seedfile)
            os.makedirs(self.seed)
            try:
                util.cmd("tar -xjpf %s -C %s/" % (seedfile, self.seed))
            except:
                shutil.rmtree(self.seed)
                raise

        util.info("Syncing %s to %s" % (self.seed.dname(), self.target_root.dname()))
        util.cmd("rsync -a --delete %s %s" % (self.seed.dname(), self.target_root.dname()))

def devtool_list_templates_test():
    templates = os.listdir(mzbench_dir + "worker_templates")
    got_templates = filter(lambda x: x, cmd(mzbench_dir + "bin/mzbench list_templates").split("\n"))
    if sorted(templates) != sorted(got_templates):
        print sorted(templates)
        print sorted(got_templates)
        assert sorted(templates) == sorted(got_templates)

	def backup(self, incremental=False):
		includes = []
		for k, group in self.sourceGroups.items():
			for item in group.items:
				includes.append('--include={0}'.format(item.path))
		
		os.environ['PASSPHRASE'] = getpwd()
		cmdList = ["duplicity", 
					"incremental" if incremental else "full",
					"--name='{0}'".format(self.targetGroup.name)]
		cmdList.extend(includes)
		cmdList.extend(["--exclude=**", "/", self.getTarget().geturl()])
		cmd(cmdList)
		
		del os.environ['PASSPHRASE']
		self.doneSomething()

def get_tabsint_plugin_docs(name, src, version=None):
    """ include tabsint plugin documentation """

    def copy_userguide(path, ext="rst"):
        installPath = "../docs/userguide/src/docs/plugins"
        try:
            os.mkdir(installPath)
        except:
            pass

        try:
            shutil.copy("{0}/index.{1}".format(path, ext), "{0}/plugin-{1}.{2}".format(installPath, name, ext))

            # copy any other ancillary files - must be named the same as the plugin!
            if name in os.listdir(path):
                shutil.copytree("{0}/{1}".format(path, name), "{0}/{1}/".format(installPath, name))
        except Exception as e:
            sys.exit('[ERROR]: Failed to copy docs for plugin "{0}" to userguide. Error: {1} '.format(name, str(e)))

    # git files
    if any(s in src for s in ["https://", ".git"]):
        [repo, subdir] = ut.checkout(src, tag=version)
        docPath = ".tmp/{0}/{1}".format(repo, subdir)

    # local files
    else:
        src = "../" + src
        ut.check_tag(src, version)

        if src.endswith("/"):
            docPath = src[:-1]
        else:
            docPath = src

    # user guide
    if "index.rst" in os.listdir(docPath):
        copy_userguide(docPath)
    elif "index.md" in os.listdir(docPath):
        copy_userguide(docPath, ext="md")

    ut.cmd("rm -rf .tmp", force=True)  # remove temp git directory, if its there

    ut.log.info('[BUILD]: Successfully retrieved docs for tabsint plugin "{0}"'.format(name))

def devtool_run_local_tests():
    run_erl_and_bdl('validate', 'loop_rate')

    run_erl_and_bdl('validate', 'env', ' --env pool_size=20 --env jozin=jozin --env wait_ms=100')

    run_erl_and_bdl('run_local', 'loop_rate')

    run_erl_and_bdl('run_local', 'data_script')

    try:
        cmd(mzbench_dir + 'bin/mzbench run_local ' + scripts_dir + 'syntax_error.erl')
        assert False
    except subprocess.CalledProcessError:
        pass

    try:
        cmd(mzbench_dir + 'bin/mzbench run_local ' + scripts_dir + 'semantic_error.erl')
        assert False
    except subprocess.CalledProcessError:
        pass

def list_plugins():
    """ retrieve list of plugins and versions"""
    try:
        [code, stdout] = ut.cmd("npm run cordova --silent -- plugins ls", suppress=True)
        plugins = [p for p in stdout.replace("\n", ",").split(",") if p != ""]
        plugins = [p.split(' "')[0] for p in plugins]
        return plugins
    except:
        sys.exit(
            "[ERROR]: Error while trying to get a list of plugins from cordova. Try running 'npm run cordova --silent -- plugins ls', or for deeper investigation see the function 'list_plugins()' in /bin/util.py"
        )

def get_tabsint_plugin(name, src, version=None):
    """ get tabsint plugin from git repository or local folder"""

    def copy_to_plugins(path):
        shutil.rmtree("../www/tabsint_plugins/{0}".format(name), True)
        shutil.copytree(path, "../www/tabsint_plugins/{0}".format(name))

    # git repos
    if any(s in src for s in ["https://", ".git"]):
        [repo, subdir] = ut.checkout(src, tag=version)
        copy_to_plugins(".tmp/{0}/{1}".format(repo, subdir))
        ut.cmd("rm -rf .tmp")  # remove temp git directory

    # local files
    else:
        src = "../" + src  # make path relative to top level directory
        ut.check_tag(src, version)
        copy_to_plugins(src)

    ut.log.info('[BUILD]: Successfully retrieved the source for tabsint plugin "{0}"'.format(name))

    def merge_kernel(self):
        args = ['--build_name', self.build_name,
            '--kernel_pkg', '\'%s\'' % (self.kernel.kernel_pkg,)]

        cmdline = '%s/kernel.sh %s' % (
            self.env['INHIBITOR_SCRIPT_ROOT'],
            ' '.join(args) )

        env = {}
        env.update(self.env)
        env['ROOT'] = '/tmp/inhibitor/kernelbuild'

        if self.seed:
            util.mkdir( self.target_root.pjoin(env['ROOT']) )
            util.chroot(
                path        = self.target_root,
                function    = util.cmd,
                fargs       = {'cmdline':cmdline, 'env':env},
                failuref    = self.chroot_failure,
            )

            # Grab any modules or firmware and put them into the embedded root fs.
            for d in ('modules', 'firmware'):
                util.chroot(
                    path        = self.target_root,
                    function    = util.cmd,
                    fargs       = {'cmdline': 'rsync -a --delete-after %s %s/' % (
                                        '/tmp/inhibitor/kernelbuild/lib/%s' % (d,),
                                        self.target_root.pjoin('lib'))},
                    failuref    = self.chroot_failure,
                )

        else:
            util.cmd( cmdline, env )

            # Grab any modules or firmware and put them into the embedded root fs.
            for d in ('modules', 'firmware'):
                util.cmd('rsync -a --delete-after %s %s/' % (
                    '/tmp/inhibitor/kernelbuild/lib/%s' % (d,),
                    self.target_root.pjoin('lib')))

def start_mzbench_server():
    if 'MZBENCH_RSYNC' in os.environ:
        node_location_param = '{{mzbench_rsync, "{0}"}}'.format(os.environ['MZBENCH_RSYNC'])
    elif 'MZBENCH_REPO' in os.environ:
        node_location_param = '{{mzbench_git, "{0}"}}'.format(os.environ['MZBENCH_REPO'])
    else:
        node_location_param = ''

    with open(dirname + "/test_server.config", "w") as config:
        config.write('[{{mzbench_api, [{0}]}}].'.format(node_location_param))

    with open('{0}/test_server.config'.format(dirname), 'r') as f:
        print(f.read())

    cmd('{0} start_server --config {1}/test_server.config'.format(mzbench_script, dirname))
    try:
        yield
    except:
        print ''
        print '-------------------- >> begin server logs << ---------------------'
        print cmd('cat ' + mzbench_dir + '/server/log/console.log').replace('\\n', '\n')
        print '-------------------- >> end server logs   << ---------------------'
        print ''
        raise
    finally:
        cmd('{0} stop_server'.format(mzbench_script))

def systemRefresh():
    """
    Up
    """
    log('UPDATING SYSTEM SPICE')

    chdir('/home/ec2-user/spicerackclient')
    cmd('git pull')
    cmd('tar -cvf /home/ec2-user/system.tar system')
    chdir('/home/ec2-user')
    
    currentversion = dget('systemversion', 0)
    currentversion = 1 + int(currentversion)
    put('systemversion', currentversion)

    with open('systemversion.txt', 'w') as f:
        f.write(str(currentversion))

    cmd('tar --append --file=system.tar systemversion.txt')

    log('UPDATED SYSTEM SPICE TO VERSION: %s' % currentversion)
    return 'success'

def run_bench(name=None, worker_package_with_default_scenario=None, nodes=None, 
        workers_per_node=None, env={}, email=None, should_fail=False, max_retries=2,
        exclusive_node_usage=False, expected_log_message_regex=None,
        check_log_function=None, post_start=None):

    email_option = ('--email=' + email) if email else ''

    if workers_per_node:
        nodes_option = '--workers_per_node ' + str(workers_per_node)
    else:
        if nodes:
            nodes_option = '--nodes ' + ','.join(nodes)
        else:
            nodes_option = '--nodes 1'

    env_option = ' '.join(('--env={0}={1}'.format(k, v)
        for k, v in env.iteritems()))

    def run():
        if 'worker_branch' in env:
            node_commit_arg = '--node_commit={0}'.format(env['worker_branch'])
        else:
            node_commit_arg = ''

        flags = ' '.join([
            '--host=localhost:4800',
            '--exclusive_node_usage=' + ('true' if exclusive_node_usage else 'false'),
            node_commit_arg,
            nodes_option,
            env_option,
            email_option])

        if name is not None:
            invocation = mzbench_dir + 'bin/mzbench ' + flags + ' start ' + name
        elif worker_package_with_default_scenario is not None:
            invocation = mzbench_dir + 'bin/mzbench ' + flags + ' start_default_scenario_of_worker ' + worker_package_with_default_scenario
        else:
            raise RuntimeError('Neither script filename nor default scenario package provided.')

        start = subprocess.Popen(shlex.split(invocation.encode('ascii')),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        start_out, start_err = start.communicate()

        try:
            bench_id = json.loads(start_out)['id']
        except Exception:
            print 'mzbench returned invalid json: \nCommand: {0}\nOutput: {1}\nStderr: {2}'.format(invocation, start_out, start_err)
            raise

        if (post_start is not None) and wait_status(bench_id, 'running', 50):
            print "Calling post start for {0}".format(bench_id)
            post_start(bench_id)

        wait = subprocess.Popen(shlex.split(
            mzbench_dir + 'bin/mzbench --host=localhost:4800 status --wait {0}'.format(bench_id)),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        wait.communicate()

        return (bench_id, wait.returncode == 0)

    attempt = 0

    while attempt < max_retries:

        print 'Attempt #{0}'.format(attempt)

        try:
            (bench_id, success) = run()
        except Exception as e:
            print "Unexpected error: {0}".format(e)
            bench_id, success = (None, False)

        if xor(success, should_fail):
            if not expected_log_message_regex and not check_log_function:
                # no need to check the log
                return bench_id

            log_cmd = mzbench_dir + 'bin/mzbench --host=localhost:4800 log {0}'.format(bench_id)
            log = cmd(log_cmd)

            re_match = None
            if expected_log_message_regex:
                if isinstance(expected_log_message_regex, str) or isinstance(expected_log_message_regex, unicode):
                    regex = re.compile(expected_log_message_regex, re.DOTALL + re.UNICODE)
                else:
                    regex = expected_log_message_regex
                re_match = regex.search(log)

            maybe_error = None
            if check_log_function:
                maybe_error = check_log_function(log)

            if not re_match or maybe_error:
                print
                if maybe_error:
                    print "Log doesn't pass custom check:\n{0}\n\n".format(maybe_error)
                if not re_match:
                    print u"Log doesn't contain expected log message '{0}':\n".format(regex.pattern)
#.........这里部分代码省略.........