def _build_presentation_stage(config, topic_set_id):
    print("Starting presentation stage for " + topic_set_id)
    for presentation_type in [item[1] for item in config.build_scripts[topic_set_id] if item[0] == 'present']:
        link_output_dir = posixpath.join(
            posixpath.dirname(config.build_scripts[topic_set_id][('link', presentation_type)]), 'out')
        _build_link_step(config=config,
                         topic_set_id=topic_set_id,
                         script=config.build_scripts[topic_set_id][('link', presentation_type)],
                         output_dir=link_output_dir,
                         synthesis_files=[x.replace('\\', '/') for x in glob(
                             posixpath.join(posixpath.dirname(config.build_scripts[topic_set_id][('resolve', None)]),
                                            'out') + '/*')],
                         catalog_files=glob(
                             posixpath.join(config.content_set_build_dir, 'catalogs') + '/*'),
                         object_files=[x.replace('\\', '/') for x in
                                       glob(posixpath.join(config.content_set_build_dir, 'objects') + '/*/*')])

        present_output_dir = posixpath.join(
            posixpath.dirname(config.build_scripts[topic_set_id][('present', presentation_type)]), 'out')
        _build_present_step(config=config,
                            topic_set_id=topic_set_id,
                            script=config.build_scripts[topic_set_id][('present', presentation_type)],
                            output_dir=present_output_dir,
                            synthesis_files=[x.replace('\\', '/') for x in glob(
                                posixpath.join(posixpath.dirname(config.build_scripts[topic_set_id][('link', presentation_type)]),
                                               'out') + '/*')],
                            toc_files=glob(posixpath.join(config.content_set_build_dir, 'tocs') + '/*'),
                            object_files=[x.replace('\\', '/') for x in
                                          glob(posixpath.join(config.content_set_build_dir, 'objects') + '/*/*')])

def mknod(path, mode=None, device=None):
  # import stat explicitly here to fix a namespace issue.
  import stat

  if mode == None:
    mode = 00600 | stat.S_IFREG

  filename = posixpath.basename(path)
  dirname  = posixpath.dirname(path)
  destdir  = _findFileFromPath(dirname)

  access(posixpath.dirname(path), os.W_OK|os.X_OK)

  if not isinstance(destdir, FakeDir):
    raise OSError(errno.ENOTDIR, '')

  if filename in destdir.getChildren():
    raise OSError(errno.EEXIST, '')

  if mode & stat.S_IFREG:
    node = FakeFile(filename, mode)
  elif mode & stat.S_IFCHR:
    node = FakeDevice(filename, 'char', mode, device['major'], device['minor'])
  elif mode & stat.S_IFBLK:
    node = FakeDevice(filename, 'block', mode, device['major'], device['minor'])
  elif mode & stat.S_IFIFO:
    node = FakeFifo(filename, mode)
  else:
    raise OSError(errno.EINVAL, 'Invalid argument')

  destdir.linkChild(filename, node)

def main():
    chapter_files, other_files = get_filenames()

    # make previous of first file and next of last file to just bring
    # back to README
    prevs = ['README.md'] + chapter_files[:-1]
    nexts = chapter_files[1:] + ['README.md']

    print("Chapter files:")
    for prevpath, thispath, nextpath in zip(prevs, chapter_files, nexts):
        # all paths should be like 'section/file.md'
        where = posixpath.dirname(thispath)
        prev = posixpath.relpath(prevpath, where)
        next_ = posixpath.relpath(nextpath, where)
        extralinks = "[Previous](%s) | [Next](%s) |\n" % (prev, next_)
        end = END_TEMPLATE.format(
            toplevel='..', extralinks=extralinks, readmeheader=where)
        update_end(thispath, end)

    print()

    print("Other files:")
    for filename in other_files:
        where = posixpath.dirname(filename)
        end = END_TEMPLATE.format(
            toplevel=posixpath.relpath('.', where),
            extralinks="", readmeheader='list-of-contents')
        update_end(filename, end)

def request_in_scope(request):
	url = request.url
	purl = urlsplit(url)
	spurl = urlsplit(Shared.starturl)	
	scope = Shared.options['scope']
	in_scope = False

	# check for scopes
	if scope == CRAWLSCOPE_DOMAIN:
		for pattern in Shared.allowed_domains:
			if re.match(pattern, purl.hostname):
				in_scope = True	
				break			

	elif scope == CRAWLSCOPE_DIRECTORY:
		if purl.hostname != spurl.hostname:
			in_scope = False
		else:
			path  = [p for p in posixpath.dirname(purl.path).split("/") if p]
			spath = [p for p in posixpath.dirname(spurl.path).split("/") if p]			
			in_scope = path[:len(spath)] == spath
		
	elif scope == CRAWLSCOPE_URL:
		in_scope = url == Shared.starturl


	# check for excluded urls
	for pattern in Shared.excluded_urls:
		if re.match(pattern, request.url):
			in_scope = False
			break

	return in_scope

 def adjust_uri(self, uri, relativeto):
     """Called from within a Mako template, avoids adjusting the
     uri if it looks like an asset specification"""
     # Don't adjust asset spec names
     isabs = os.path.isabs(uri)
     if (not isabs) and (':' in uri):
         return uri
     if not(isabs) and ('$' in uri):
         return uri.replace('$', ':')
     if relativeto is not None:
         relativeto = relativeto.replace('$', ':')
         if not(':' in uri) and (':' in relativeto):
             if uri.startswith('/'):
                 return uri
             pkg, relto = relativeto.split(':')
             _uri = posixpath.join(posixpath.dirname(relto), uri)
             return '{0}:{1}'.format(pkg, _uri)
         if not(':' in uri) and not(':' in relativeto):
             return posixpath.join(posixpath.dirname(relativeto), uri)
     return TemplateLookup.adjust_uri(
         self,
         uri,
         relativeto,
         input_encoding='utf-8'
     )

def iso_country_to_countryball(isocode):
    """returns the countryball for given isocode
    omsk if file not found"""
    if isocode is None:
        return 'unknown.png'

    if isocode == 'BAY':
        isocode = 'bavaria'
    elif isocode == 'TEX':
        isocode = 'texas'

    isocode = isocode.lower()

    # rather dirty hack to get the path
    basepath = os.path.join(dirname(dirname(__file__)), 'static', 'img', 'cb')

    if rfk.CONFIG.has_option('site', 'cbprefix'):
        prebasepath = os.path.join(basepath, rfk.CONFIG.get('site', 'cbprefix'))
        if os.path.exists(os.path.join(prebasepath, '{}.png'.format(isocode))):
            return '{}{}.png'.format(rfk.CONFIG.get('site', 'cbprefix'), isocode)

    if os.path.exists(os.path.join(basepath, '{}.png'.format(isocode))):
        return '{}.png'.format(isocode)
    else:
        return 'unknown.png'

    def validate_link(self, url, link):
        # Remove anchor
        link = re.sub(r'#[^#]*$', '', link)

        # Skip prefix
        if re.search('^(#|javascript:|mailto:|tel:)', link):
            return None

        #validate URL
        rx_url = re.match('(https?://)([^/:]+)(:[0-9]+)?([^\?]*)(\?.*)?', url)
        url_protocol = rx_url.group(1)
        url_host = rx_url.group(2)
        url_port = rx_url.group(3) if rx_url.group(3) else ''
        url_path = rx_url.group(4) if len(rx_url.group(4)) > 0 else '/'
        url_dir_path = dirname(url_path)

        #validate link and create a full url using above 'url'
        rx_link = re.match('((https?://)([^/:]+)(:[0-9]+)?)?([^\?]*)(\?.*)?', link)
        link_full_url = rx_link.group(1) != None
        link_protocol = rx_link.group(2) if rx_link.group(2) else url_protocol
        link_host = rx_link.group(3) if rx_link.group(3) else url_host
        link_port = rx_link.group(4) if rx_link.group(4) else url_port
        link_path = quote(rx_link.group(5), '/%') if rx_link.group(5) else url_path
        link_query = quote(rx_link.group(6), '?=&%') if rx_link.group(6) else ''
        link_dir_path = dirname(link_path)

        if not link_full_url and not link.startswith('/'):
            link_path = normpath(join(url_dir_path, link_path))

        link_url = link_protocol + link_host + link_port + link_path + link_query
        return link_url

 def parent_dir(self):
     if self.file_name:
         return posixpath.dirname(self._resource_path) + "/"
     elif self.is_root:
         return self._resource_path
     else:
         return posixpath.dirname(posixpath.dirname(self._resource_path)) + "/"

    def create_file(self, request=None):
        # Build an index page
        import tempfile
        fd, tempname = tempfile.mkstemp()

        # FIXME: Shouldn't hardcode this
        urlroot = "/music/"

        if request:
            inet, addr, port = request.getHost()
            if port == 80:
                hostport = ''
            else:
                hostport = ':%d' % port

            import urllib

            server =  urllib.quote('http%s://%s%s' % (
                request.isSecure() and 's' or '',
                request.getRequestHostname(),
                hostport), "/:")
            path = request.path[len(urlroot):]

        else:
            server = "http://unknown.org"
            path = self.cwd()

        os.write(fd, '<?xml version="1.0" encoding="utf-8"?>\n')

        cnt = self.select()
        os.write(fd, '<%s length="%s">\n' % (self.document_type, cnt))

        # Strip away "list/<stylesheet>", if present
        if posixpath.basename(posixpath.dirname(path)) == "list":
            path = posixpath.dirname(posixpath.dirname(path)) + "/"
        path = xml_fix_string(path).replace("%20", ' ') or '/'
        if path != '/':
            path = "".join([ (elem and "<d>%s</d>" % elem)
                             for elem in path.split("/") ])

        os.write(fd, '  <path>%s</path>\n' % path)

        self.write_body(fd)

        os.write(fd, "</%s>\n" % self.document_type)
        os.close(fd)
        self.cursor.close()
        self.cursor = None

        # Perform xslt transformation on the file
        from commands import getstatusoutput
        params = "--stringparam audiostore.root %s" % urlroot
        params += " --stringparam audiostore.url %s%s" % (server, urlroot)
        logger.info("xsltproc %s %s %s" % \
                    (params, self.xsltfile, tempname))
        st, output = getstatusoutput("xsltproc %s %s %s" % \
                                     (params, self.xsltfile, tempname))

        self.file = StringIO(output)

    def __init__(self, json_data):
        build_settings = json_data["build_settings"]
        self.root_path = build_settings["root_path"]
        self.build_dir = build_settings["build_dir"]
        self.default_toolchain = build_settings["default_toolchain"]

        self.targets = {}
        for key, value in json_data["targets"].items():
            self.targets[key] = Target(key, value, self)

        # Build files are a bit special; we load all build files that gn know about
        # Build files belonging to target folders will be added to respective target sources
        # For build files in //build/ folder we load all surrounding files, as the .gn and .gni
        # files may include scripts and resources gn does not known about
        self.build_files = []
        with open(self.get_absolute_build_path() + "build.ninja.d", "r") as f:
            l = f.readline()
            args = l.split(" ")
            root_path = self.root_path
            if not root_path.endswith("/"):
                root_path += "/"

            known_build_files = set(args)
            processed_dirs = set()

            for arg in args:
                if arg.startswith(self.root_path):
                    short_name = "//" + arg[len(root_path):]
                    self.build_files.append(short_name)
                    dir = posixpath.dirname(arg)
                    if short_name.startswith("//build/") and not dir in processed_dirs:
                        processed_dirs.add(dir)

                        for file in os.listdir(dir):
                            ext = posixpath.splitext(file)[1]
                            if ext == ".pyc":
                                continue
                            path = dir + "/" + file
                            if not path in known_build_files and os.path.isfile(path):
                                short_name = "//" + path[len(root_path):]
                                self.build_files.append(short_name)
                                known_build_files.add(path)

        # Go through targets and add build files belonging to those
        for target_name, target in self.targets.items():
            source_dir = target.get_source_dir()
            for file in self.build_files:
                path = posixpath.dirname(file) + "/"
                if source_dir == path:
                    target.sources.append(file)

        # create build target
        build_target = Target("//build:build", {"type" : "build_dir", "toolchain" : self.default_toolchain}, self)
        for build_file in self.build_files:
            if (build_file.startswith(build_target.get_source_dir()) or
                posixpath.dirname(build_file) == "//" or # Also add root files to build dir
                build_file == self.build_dir + "args.gn"):
                build_target.sources.append(build_file)
        self.targets[build_target.name] = build_target

    def rename(self, src, dst):
        """
        Rename a file/directory from src to dst.

        Raises OSError on error.
        """
        src = self.abspath(src)
        dst = self.abspath(dst)
        logging.debug("rename %r -> %r" % (src, dst))
        self._listdir_cache.flush()
        # Check not renaming to itself
        if src == dst:
            logging.debug("Renaming %r to itself - doing nothing" % src)
            return
        # If dst is an existing directory, copy src inside it
        if self.isdir(dst):
            if dst:
                dst += "/"
            dst += posixpath.basename(src)
        # Check constraints for renaming a directory
        if self.isdir(src):
            if self.listdir(src):
                raise IOSError(ENOTEMPTY, "Can't rename non-empty directory: %s" % src)
            if self.isfile(dst):
                raise IOSError(ENOTDIR, "Can't rename directory to file")
        # Check not renaming to itself
        if src == dst:
            logging.debug("Renaming %r to itself - doing nothing" % src)
            return
        # Parse the paths now
        src_container_name, src_path = parse_fspath(src)
        dst_container_name, dst_path = parse_fspath(dst)
        logging.debug("`.. %r/%r -> %r/%r" % (src_container_name, src_path, dst_container_name, dst_path))
        # Check if we are renaming containers
        if not src_path and not dst_path and src_container_name and dst_container_name:
            return self._rename_container(src_container_name, dst_container_name)
        # ...otherwise can't deal with root stuff
        if not src_container_name or not src_path or not dst_container_name or not dst_path:
            raise IOSError(EACCES, "Can't rename to / from root")
        # Check destination directory exists
        if not self.isdir(posixpath.split(dst)[0]):
            raise IOSError(ENOENT, "Can't copy %r to %r, destination directory doesn't exist" % (src, dst))

        # check dst container
        self._container_exists(dst_container_name)

        # Do the rename of the file/dir
        meta = self.conn.head_object(src_container_name, src_path)
        if 'x-object-manifest' in meta:
            # a manifest file
            headers = { 'x-object-manifest': quote(meta['x-object-manifest']) }
        else:
            # regular file
            headers = { 'x-copy-from': quote("/%s/%s" % (src_container_name, src_path)) }
        self.conn.put_object(dst_container_name, dst_path, headers=headers, contents=None)
        # Delete src
        self.conn.delete_object(src_container_name, src_path)
        self._listdir_cache.flush(posixpath.dirname(src))
        self._listdir_cache.flush(posixpath.dirname(dst))

    def test_dirname(self):
        self.assertEqual(posixpath.dirname("/foo/bar"), "/foo")
        self.assertEqual(posixpath.dirname("/"), "/")
        self.assertEqual(posixpath.dirname("foo"), "")
        self.assertEqual(posixpath.dirname("////foo"), "////")
        self.assertEqual(posixpath.dirname("//foo//bar"), "//foo")

        self.assertRaises(TypeError, posixpath.dirname)

def am_gem(fd, var, gem, am):
    gemre = re.compile(r'\.files *= *\[ *(.*[^ ]) *\]')
    rd = 'RUBY_DIR'
    if 'DIR' in gem:
        rd = gem['DIR'][0]
    rd = am_translate_dir(rd, am)
    fd.write('if HAVE_RUBYGEM\n')
    fd.write('all-local-%s:' % var)
    am['ALL'].append(var)
    for f in gem['FILES']:
        fd.write(' %s' % f[:-4])
    fd.write('\n')
    for f in gem['FILES']:
        srcs = list(map(lambda x: x.strip('" '),
                   gemre.search(open(os.path.join(am['CWDRAW'], f)).read()).group(1).split(', ')))
        srcs.append(f)
        sf = f.replace('.', '_')
        am['INSTALL'].append(sf)
        am['UNINSTALL'].append(sf)
        fd.write('%s: %s\n' % (f[:-4], ' '.join(srcs)))
        dirs = []
        for src in srcs:
            if '/' in src:
                d = posixpath.dirname(src)
                if d not in dirs:
                    fd.write("\t[ '$(srcdir)' -ef . ] || mkdir -p '%s'\n" % posixpath.dirname(src))
                    dirs.append(d)
                    while '/' in d:
                        d = posixpath.dirname(d)
                        dirs.append(d)
            fd.write("\t[ '$(srcdir)' -ef . ] || cp -p '$(srcdir)/%s' '%s'\n" % (src, src))
        fd.write("\tgem build '%s'\n" % f)
        # use deprecated --rdoc and --ri options instead of --document=rdoc,ri
        # since we're still building on systems with old gem
        fd.write("\tgem install --local --install-dir ./'%s' --bindir .'%s' --force --rdoc --ri %s\n" % (rd, am_translate_dir('bindir', am), f[:-4]))
        fd.write('mostlyclean-local: mostlyclean-local-%s\n' % sf)
        fd.write('.PHONY: mostlyclean-local-%s\n' % sf)
        fd.write('mostlyclean-local-%s:\n' % sf)
        for src in srcs:
            fd.write("\t[ '$(srcdir)' -ef . ] || rm -f '%s'\n" % src)
        for d in sorted(dirs, reverse = True):
            fd.write("\t[ '$(srcdir)' -ef . -o ! -d '%s' ] || rmdir '%s'\n" % (d, d))
        fd.write("install-exec-local-%s: %s\n" % (sf, f[:-4]))
        fd.write("\tmkdir -p $(DESTDIR)'%s'\n" % rd)
        fd.write("\tcp -a ./'%s'/* $(DESTDIR)'%s'\n" % (rd, rd))
        fd.write("uninstall-local-%s: %s\n" % (sf, f[:-4]))
        # remove "-0.1.gemspec" from end of `f'
        fd.write("\tgem uninstall --install-dir $(DESTDIR)'%s' '%s'\n" % (rd, f[:-12]))
        am['BUILT_SOURCES'].append(f[:-4])
        am['CLEAN'].append(f[:-4])
    fd.write('else\n')
    for f in gem['FILES']:
        sf = f.replace('.', '_')
        fd.write("install-exec-local-%s:\n" % sf)
        fd.write('uninstall-local-%s:\n' % sf)
    fd.write('endif\n')

    def _ensure_parent_dirs(self):
        curdir = vcspath.dirname(self.node['path'])
        dirs_to_create = []
        while not self._svn_path_exists(curdir):
            dirs_to_create.append(curdir)
            curdir = vcspath.dirname(curdir)

        for curdir in reversed(dirs_to_create):
            log.debug('Creating missing directory "%s"', curdir)
            svn.fs.make_dir(self.txn_root, curdir)

def getEntryName(path):
    """
    Retrieve the top level name (not h5py object) associated to a given path
    despite being or not an NXentry group.
    """
    entry = path
    candidate = posixpath.dirname(entry)
    while len(candidate) > 1:
        entry = candidate
        candidate = posixpath.dirname(entry)
    return entry

  def _url_dirname(url_or_path):
    """Like posixpath.dirname, but preserves scheme:// prefix.

    Args:
      url_or_path: A string in the form of scheme://some/path OR /some/path.
    """
    match = re.match(r'([a-z]+://)(.*)', url_or_path)
    if match is None:
      return posixpath.dirname(url_or_path)
    url_prefix, path = match.groups()
    return url_prefix + posixpath.dirname(path)

def create_udf_data(con):
    ibis_home = posixpath.dirname(posixpath.dirname(os.path.abspath(__file__)))
    sep = os.sep
    path_list = ibis_home.split(sep)
    path_list += ['testing', 'udf']
    udf_dir = sep.join(path_list)
    build_list = path_list + ['build']
    build_dir = sep.join(build_list)
    subprocess.check_call('cmake . && make', shell=True, cwd=udf_dir)
    so_dir = pjoin(ENV.test_data_dir, 'udf')
    con.hdfs.put(so_dir, build_dir, verbose=True)

    def test_dirname(self):
        self.assertEqual(posixpath.dirname("/foo/bar"), "/foo")
        self.assertEqual(posixpath.dirname("/"), "/")
        self.assertEqual(posixpath.dirname("foo"), "")
        self.assertEqual(posixpath.dirname("////foo"), "////")
        self.assertEqual(posixpath.dirname("//foo//bar"), "//foo")

        self.assertEqual(posixpath.dirname(b"/foo/bar"), b"/foo")
        self.assertEqual(posixpath.dirname(b"/"), b"/")
        self.assertEqual(posixpath.dirname(b"foo"), b"")
        self.assertEqual(posixpath.dirname(b"////foo"), b"////")
        self.assertEqual(posixpath.dirname(b"//foo//bar"), b"//foo")

def iso_country_to_countryball(isocode):
    """returns the countryball for given isocode
    omsk if file not found"""
    if isocode is None:
        return 'unknown.png'
    isocode = isocode.lower()
    #rather dirty hack to get the path
    basepath = os.path.join(dirname(dirname(__file__)), 'static', 'img', 'cb')
    if os.path.exists(os.path.join(basepath,'{}.png'.format(isocode))):
        return '{}.png'.format(isocode)
    else:
        return 'unknown.png'

 def rename(self, src, dst):
     '''Rename a file/directory from src to dst, raise OSError on error'''
     src = self.abspath(src)
     dst = self.abspath(dst)
     logging.debug("rename %r -> %r" % (src, dst))
     self._listdir_cache.flush()
     # Check not renaming to itself
     if src == dst:
         logging.debug("Renaming %r to itself - doing nothing" % src)
         return
     # If dst is an existing directory, copy src inside it
     if self.isdir(dst):
         if dst:
             dst += "/"
         dst += posixpath.basename(src)
     # Check constraints for renaming a directory
     if self.isdir(src):
         if self.listdir(src):
             raise IOSError(ENOTEMPTY, "Can't rename non-empty directory: '%s'" % src)
         if self.isfile(dst):
             raise IOSError(ENOTDIR, "Can't rename directory to file")
     # Check not renaming to itself
     if src == dst:
         logging.debug("Renaming %r to itself - doing nothing" % src)
         return
     # Parse the paths now
     src_container_name, src_path = parse_fspath(src)
     dst_container_name, dst_path = parse_fspath(dst)
     logging.debug("`.. %r/%r -> %r/%r" % (src_container_name, src_path, dst_container_name, dst_path))
     # Check if we are renaming containers
     if not src_path and not dst_path and src_container_name and dst_container_name:
         return self._rename_container(src_container_name, dst_container_name)
     # ...otherwise can't deal with root stuff
     if not src_container_name or not src_path or not dst_container_name or not dst_path:
         logging.info("Can't rename %r -> %r" % (src, dst))
         raise IOSError(EACCES, "Can't rename to / from root")
     # Check destination directory exists
     if not self.isdir(posixpath.split(dst)[0]):
         logging.info("Can't copy %r -> %r dst directory doesn't exist" % (src, dst))
         raise IOSError(ENOENT, "Can't copy %r -> %r dst directory doesn't exist" % (src, dst))
     # Do the rename of the file/dir
     src_container = self._get_container(src_container_name)
     dst_container = self._get_container(dst_container_name)
     src_obj = src_container.get_object(src_path)
     # Copy src -> dst
     src_obj.copy_to(dst_container_name, dst_path)
     # Delete dst
     src_container.delete_object(src_path)
     self._listdir_cache.flush(posixpath.dirname(src))
     self._listdir_cache.flush(posixpath.dirname(dst))