def findT(path, language="en-us"):
    """
    must be run by the admin app
    """
    filename = os.path.join(path, "languages", "%s.py" % language)
    sentences = read_dict(filename)
    mp = os.path.join(path, "models")
    cp = os.path.join(path, "controllers")
    vp = os.path.join(path, "views")
    for file in listdir(mp, ".+\.py", 0) + listdir(cp, ".+\.py", 0) + listdir(vp, ".+\.html", 0):
        fp = open(file, "r")
        portalocker.lock(fp, portalocker.LOCK_SH)
        data = fp.read()
        portalocker.unlock(fp)
        fp.close()
        items = regex_translate.findall(data)
        for item in items:
            try:
                message = eval(item)
                if not message.startswith("#") and not "\n" in message:
                    tokens = message.rsplit("##", 1)
                else:
                    # this allows markmin syntax in translations
                    tokens = [message]
                if len(tokens) == 2:
                    message = tokens[0].strip() + "##" + tokens[1].strip()
                if message and not message in sentences:
                    sentences[message] = message
            except:
                pass
    write_dict(filename, sentences)

def findT(path, language='en-us'):
    """
    must be run by the admin app
    """
    filename = os.path.join(path, 'languages', '%s.py' % language)
    sentences = read_dict(filename)
    mp = os.path.join(path, 'models')
    cp = os.path.join(path, 'controllers')
    vp = os.path.join(path, 'views')
    for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\
         + listdir(vp, '.+\.html', 0):
        fp = open(file, 'r')
        portalocker.lock(fp, portalocker.LOCK_SH)
        data = fp.read()
        portalocker.unlock(fp)
        fp.close()
        items = regex_translate.findall(data)
        for item in items:
            try:
                message = eval(item)
                if not message.startswith('#') and not '\n' in message:
                    tokens = message.rsplit('##', 1)
                else:
                    # this allows markmin syntax in translations
                    tokens = [message]
                if len(tokens) == 2:
                    message = tokens[0].strip() + '##' + tokens[1].strip()
                if message and not message in sentences:
                    sentences[message] = message
            except:
                pass
    write_dict(filename, sentences)

 def get_logs(self, path, last_clock):
     import portalocker
     import tailer
     import time
     pn = self.get_path(path)
     with open(pn, "r+") as log:
         while True:
             try:
                 portalocker.lock(log, portalocker.LOCK_EX)
                 break
             except:
                 dbg.dbg("lock failed")
                 time.sleep(0.1)
         curtime = int(util.current_sec())
         lines = tailer.tail(log, 20)
     ret = []
     if last_clock is None: last_clock = 0
     for line in lines:
         sp = line.strip().split("\t")
         if(int(sp[0]) < last_clock): continue
         #log = {
         #    'time': eval(sp[0]),
         #    'message': sp[1]
         #}
         #ret.append(log)
         ret.append(sp[1])
     return ret, curtime

    def __init__(self, request, folder=None):
        self.request = request

        # Lets test if the cache folder exists, if not
        # we are going to create it
        folder = folder or os.path.join(request.folder, "cache")

        if not os.path.exists(folder):
            os.mkdir(folder)

        ### we need this because of a possible bug in shelve that may
        ### or may not lock
        self.locker_name = os.path.join(request.folder, "cache/cache.lock")
        self.shelve_name = os.path.join(request.folder, "cache/cache.shelve")

        locker, locker_locked = None, False
        try:
            locker = open(self.locker_name, "a")
            portalocker.lock(locker, portalocker.LOCK_EX)
            locker_locked = True
            storage = shelve.open(self.shelve_name)

            if not storage.has_key(CacheAbstract.cache_stats_name):
                storage[CacheAbstract.cache_stats_name] = {"hit_total": 0, "misses": 0}
                storage.sync()
        except ImportError:
            pass  # no module _bsddb, ignoring exception now so it makes a ticket only if used
        except:
            logger.error("corrupted file: %s" % self.shelve_name)
        if locker_locked:
            portalocker.unlock(locker)
        if locker:
            locker.close()

 def get(self, count: int=1, readOnly: bool=False, recurlvl=0):
     with open(self.FILE, "r+") as file:
         portalocker.lock(file, portalocker.LOCK_EX)
         ports = []
         while len(ports) < count:
             file.seek(0)
             port = int(file.readline())
             if readOnly:
                 return port
             port += 1
             if port > self.maxPort:
                 port = self.minPort
             file.seek(0)
             file.write(str(port))
             try:
                 checkPortAvailable(("", port))
                 ports.append(port)
                 self.logger.debug("new port dispensed: {}".format(port))
             except Exception:
                 if recurlvl < self.maxportretries:
                     self.logger.debug("port {} unavailable, trying again...".
                                       format(port))
                     recurlvl += 1
                 else:
                     self.logger.debug("port {} unavailable, max retries {} "
                                       "reached".
                                       format(port, self.maxportretries))
                     raise
         return ports

    def __init__(self, prefix = "omero"):
        """
        Initializes a TempFileManager instance with a userDir containing
        the given prefix value, or "omero" by default. Also registers
        an atexit callback to call self.cleanup() on exit.
        """
        self.logger = logging.getLogger("omero.util.TempFileManager")
        self.is_win32 = ( sys.platform == "win32" )
        self.prefix = prefix

        self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username()))
        """
        User-accessible directory of the form $TMPDIR/omero_$USERNAME.
        If the given directory is not writable, an attempt is made
        to use an alternative
        """
        if not self.create(self.userdir) and not self.access(self.userdir):
            i = 0
            while i < 10:
                t = path("%s_%s" % (self.userdir, i))
                if self.create(t) or self.access(t):
                    self.userdir = t
                    break
            raise Exception("Failed to create temporary directory: %s" % self.userdir)
        self.dir = self.userdir / self.pid()
        """
        Directory under which all temporary files and folders will be created.
        An attempt to remove a path not in this directory will lead to an
        exception.
        """

        # Now create the directory. If a later step throws an
        # exception, we should try to rollback this change.
        if not self.dir.exists():
            self.dir.makedirs()
        self.logger.debug("Using temp dir: %s" % self.dir)

        self.lock = None
        try:
            self.lock = open(str(self.dir / ".lock"), "a+")
            """
            .lock file under self.dir which is used to prevent other
            TempFileManager instances (also in other languages) from
            cleaning up this directory.
            """
            try:
                portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB)
                atexit.register(self.cleanup)
            except:
                lock = self.lock
                self.lock = None
                if lock:
                    self.lock.close()
                raise
        finally:
            try:
                if not self.lock:
                    self.cleanup()
            except:
                self.logger.warn("Error on cleanup after error", exc_info = True)

	def lock(self,key):

		fname = self.__fname(key)

		prelock = int(self.__filemtime(fname))

		fp = open(fname, "ab")
		
		self.open[key] = fp

		os.chmod(fname, 0600)

		portalocker.lock(fp,portalocker.LOCK_EX)


		# Written while blocking ?
		if(prelock > 0 and self.__filemtime(fname) > prelock):
			if(self.logfile): self.__log(key+" prelock: " + str(prelock) + " postlock: "+str(self.__filemtime(fname))+" NOT writing.")
			self.unlock(key)
			del self.open[key]
			return None
		

		if(self.logfile):
			 self.__log(key+" prelock: " + str(prelock) + " postlock: "+ str(self.__filemtime(fname)) +" writing.")

		self.locked[key] = fp

		return True

 def lock(self):
     if not self.locked:
         portalocker.lock(self._f, portalocker.LOCK_EX)
         self.locked = True
         return True
     else:
         return False

def load_module(name, code=None, name_path=""):
    # http://stackoverflow.com/a/30407477/5288758
    try:
        import importlib
    except ImportError:
        import imp as importlib

    if code is not None:
        try:
            # Try and create/open the file only if it doesn't exist.
            fd = os.open(name, os.O_CREAT | os.O_EXCL | os.O_WRONLY)

            # Lock the file exclusively to notify other processes we're still writing
            portalocker.lock(fd, portalocker.LOCK_EX)  # fcntl.flock(fd, fcntl.LOCK_EX)
            with os.fdopen(fd, 'w') as f:
                f.write(code)

        except OSError as e:
            # If the error wasn't EEXIST we should raise it.
            if e.errno != errno.EEXIST:
                raise

    # The file existed, so let's open it for reading and then try and
    # lock it. This will block on the LOCK_EX above if it's held by
    # the writing process.
    with open(name, "r") as f:
        portalocker.lock(f, portalocker.LOCK_EX)  # fcntl.flock(f, fcntl.LOCK_EX)

    return importlib.import_module(name, name_path)

	def __getCachedData(self,key):

		fname = self.__fname(key)

		# Pre-existing locked read
		if(self.locked.get(key)):
			self.locked[key].seek(0)
			try: str = pickle.load(self.locked[key])
			except: str = False
			self.locked[key].seek(0)
			return str
		

		fp=open(fname, "r")
		self.open[key] = fp
		portalocker.lock(fp,portalocker.LOCK_SH)
		
		# The following 2 lines handle cases where open (above) was called
		# on an empty file that was created by cache::lock()
		fp.seek(0)
		
		try: str = pickle.load(fp)
		except: str = False
		
		try: portalocker.unlock(fp)
		except: print "Cache error unlocking file with key " + key
		try: fp.close()
		except: print "Cache error closing file with key " + key
			
		del self.open[key]
		return str

 def _open_shelve_and_lock(self):
     """Open and return a shelf object, obtaining an exclusive lock
     on self.locker first. Replaces the close method of the
     returned shelf instance with one that releases the lock upon
     closing."""
     
     storage = None
     locker = None
     locked = False
     try:
         locker = locker = open(self.locker_name, 'a')
         portalocker.lock(locker, portalocker.LOCK_EX)
         locked = True
         try:
             storage = shelve.open(self.shelve_name)
         except:
             logger.error('corrupted cache file %s, will try rebuild it' \
                              % (self.shelve_name))
             storage = None
         if not storage and os.path.exists(self.shelve_name):
             os.unlink(self.shelve_name)
             storage = shelve.open(self.shelve_name)
         if not CacheAbstract.cache_stats_name in storage.keys():
             storage[CacheAbstract.cache_stats_name] = {'hit_total':0, 'misses': 0}
         storage.sync()
     except Exception, e:
         if storage:
             storage.close()
             storage = None
         if locked:
             portalocker.unlock(locker)
             locker.close()
         locked = False
         raise RuntimeError, 'unable to create/re-create cache file %s' % self.shelve_name

def portalock_open(myfilename):
    myfile = open(myfilename,'r+') # open in rw mode, never write mode
    ## both LOCK_EX and LOCK_SH wait indefinitely for lock to get acquired
    ## i.e. if other process has locked file, wait indefinitely
    ## till lock is released by other process
    portalocker.lock(myfile,portalocker.LOCK_EX) # try to acquire lock
    return myfile

 def __call__(
     self,
     key,
     f,
     time_expire = DEFAULT_TIME_EXPIRE,
     ):
     dt = time_expire
     locker = open(self.locker_name,'a')
     portalocker.lock(locker, portalocker.LOCK_EX)
     storage = shelve.open(self.shelve_name)
     item = storage.get(key, None)
     if item and f == None:
         del storage[key]
     portalocker.unlock(locker)
     locker.close()
     if f is None:
         return None
     if item and (dt == None or item[0] > time.time() - dt):
         return item[1]
     value = f()
     locker = open(self.locker_name,'a')
     portalocker.lock(locker, portalocker.LOCK_EX)
     storage[key] = (time.time(), value)
     storage.sync()
     portalocker.unlock(locker)
     locker.close()
     return value

def parseNewWorm(wormID, name):
	global worms

	name = name.replace("\t", " ").strip() # Do not allow tab in names, it will screw up our ranking tab-separated text-file database
	exists = False
	try:
		worm = worms[wormID]
		exists = True
	except KeyError: #Worm doesn't exist.
		worm = Worm()
	worm.Name = name
	worm.iID = wormID
	worm.Ping = []

	worms[wormID] = worm

	if io.getGameType() == "Hide and Seek":
		minSeekers = 1
		if len(worms.values()) >= 4: minSeekers = 2
		if io.getNumberWormsInTeam(1) < minSeekers:
			io.setWormTeam(wormID, 1) # Seeker
		else:
			io.setWormTeam(wormID, 0) # Hider		
	else:
		# Balance teams
		teams = [0,0,0,0]
		for w in worms.keys():
			teams[worms[w].Team] += 1
		minTeam = 0
		minTeamCount = teams[0]
		for f in range(cfg.MAX_TEAMS):
			if minTeamCount > teams[f]:
				minTeamCount = teams[f]
				minTeam = f

		io.setWormTeam(wormID, minTeam)

	if cfg.RANKING_AUTHENTICATION:
		if not name in ranking.auth:
			ranking.auth[name] = getWormSkin(wormID)
			try:
				f = open(io.getFullFileName("pwn0meter_auth.txt"),"r")
				try:
					portalocker.lock(f, portalocker.LOCK_EX)
				except:
					pass
				f.write( name + "\t" + str(ranking.auth[name][0]) + " " + ranking.auth[name][1] + "\n" )
				f.close()
			except IOError:
				msg("ERROR: Unable to open pwn0meter_auth.txt")
		else:
			if ranking.auth[name] != getWormSkin(wormID):
				io.kickWorm(wormID, "Player with name %s already registered" % name)

	wormIP = io.getWormIP(wormID).split(":")[0]
	# io.messageLog("Curtime " + str(time.time()) + " IP " + str(wormIP) + " Kicked worms: " + str(cmds.kickedUsers), io.LOG_INFO)
	if wormIP in cmds.kickedUsers and cmds.kickedUsers[ wormIP ] > time.time():
			io.kickWorm( wormID, "You can join in " + str(int(cmds.kickedUsers[ wormIP ] - time.time())/60 + 1) + " minutes" )
			return
	cmds.recheckVote()

    def _write_to_file(self, params, write_callback):
        full_dir = res_template(self.dir_template, params)
        dir_key = self.get_dir_key(params)

        result = self.fh_cache.get(dir_key)

        close_file = False

        new_size = start = 0

        if result:
            out, filename = result
            is_new = False
        else:
            filename = self.get_new_filename(full_dir, params)

            if not self.allow_new_file(filename, params):
                return False

            out = self._open_file(filename, params)

            is_new = True

        try:
            start = out.tell()

            write_callback(out, filename)

            out.flush()

            new_size = out.tell()

            out.seek(start)

            if self.dedup_index:
                self.dedup_index.add_urls_to_index(out, params,
                                                   filename,
                                                   new_size - start)

            return True

        except Exception as e:
            traceback.print_exc()
            close_file = True
            return False

        finally:
            # check for rollover
            if self.max_size and new_size > self.max_size:
                close_file = True

            if close_file:
                self._close_file(out)
                if not is_new:
                    self.fh_cache.pop(dir_key, None)

            elif is_new:
                if os.name != 'nt':
                    portalocker.lock(out, portalocker.LOCK_EX | portalocker.LOCK_NB)
                self.fh_cache[dir_key] = (out, filename)

    def acquire( self, wait = 5 * 60, waitInterval = 5, expire = 10 * 60 ):
        waitTotal = 0
        while ( self.checkValidLock() ):
            if ( wait is None ):
                raise Exception( 'lock is busy' )
            else:
                if ( wait != 0 ):
                    waitTotal += waitInterval
                    if ( self.debug ):
                        print( 'waitTotal: %d wait: %d waitInterval: %d' % ( waitTotal, wait, waitInterval ) )
                    if ( waitTotal > wait ):
                        raise Exception( 'exceeded max wait time on the lock' )
                    time.sleep( waitInterval )

        # don't want blocking on acquired locks - even with the loop, there is still a possibility of stolen lock and exception here
        self.handle = file( self.lockfile, 'w' )
        portalocker.lock( self.handle, portalocker.LOCK_EX | portalocker.LOCK_NB )
        if ( self.debug ):
            print( 'acquired lock %s' % self.lockfile )
        pickle.dump( os.getpid(), self.handle )
        if ( expire is None ):
            expire_time = None
        else:
            expire_time = datetime.datetime.now()
            expire_time += datetime.timedelta( seconds = expire )
        pickle.dump( expire_time, self.handle )
        pickle.dump( self.lockinfo, self.handle )
        self.handle.flush()

    def acquire(self):
        """ Acquire thread and file locks.  Re-opening log for 'degraded' mode.
        """
        # handle thread lock
        if Handler:
            # under some tests Handler ends up being null due to instantiation
            # order
            Handler.acquire(self)

        # Issue a file lock.  (This is inefficient for multiple active threads
        # within a single process. But if you're worried about high-performance,
        # you probably aren't using this log handler.)
        if self.stream_lock:
            # If stream_lock=None, then assume close() was called or something
            # else weird and ignore all file-level locks.
            if self.stream_lock.closed:
                # Daemonization can close all open file descriptors, see
                # https://bugzilla.redhat.com/show_bug.cgi?id=952929
                # Try opening the lock file again.  Should we warn() here?!?
                try:
                    self._open_lockfile()
                except Exception:
                    self.handleError(NullLogRecord())
                    # Don't try to open the stream lock again
                    self.stream_lock = None
                    return
            lock(self.stream_lock, LOCK_EX)

def load_storage(filename):
    fp = open(filename, 'rb')
    portalocker.lock(fp, portalocker.LOCK_EX)
    storage = cPickle.load(fp)
    portalocker.unlock(fp)
    fp.close()
    return Storage(storage)

    def from_conf(cls, path=None, **overrides):
        '''Initialize instance from YAML configuration file,
            writing updates (only to keys, specified by "conf_update_keys") back to it.'''
        import yaml

        if path is None:
            path = cls.conf_path_default
            log.debug('Using default state-file path: {}'.format(path))
        path = os.path.expanduser(path)
        with open(path, 'r') as src:
            portalocker.lock(src, portalocker.LOCK_SH)
            # fcntl.lockf(src, fcntl.LOCK_SH)
            conf = yaml.load(src.read())
            portalocker.unlock(src)
        conf.setdefault('conf_save', path)

        conf_cls = dict()
        for ns, keys in cls.conf_update_keys.viewitems():
            for k in keys:
                try:
                    v = conf.get(ns, dict()).get(k)
                except AttributeError:
                    if not cls.conf_raise_structure_errors: raise
                    raise KeyError('Unable to get value for configuration parameter'
                                   ' "{k}" in section "{ns}", check configuration file (path: {path}) syntax'
                                   ' near the aforementioned section/value.'.format(ns=ns, k=k, path=path))
                if v is not None:
                    conf_cls['{}_{}'.format(ns, k)] = conf[ns][k]
        conf_cls.update(overrides)

        self = cls(**conf_cls)
        self.conf_save = conf['conf_save']
        return self

def planet():
    #return ""
    import gluon.contrib.rss2 as rss2

    # store planet rss entries in disk (forever...)
    import portalocker
    import os, cPickle as pickle
    path = os.path.join(request.folder,'cache', "planet.rss")
    if not os.path.exists(path):
        f = open(path, "w+")
        rss = get_planet_rss(None)
        rss = [{'title': item.title, 'author': item.author, 'pubDate': item.pubDate, 'link': item.link, 'description': item.description} for item in rss.items]
    else:
        f = open(path, "r+")
        rss = None
    portalocker.lock(f, portalocker.LOCK_EX)
    if not rss:
        rss = pickle.load(f)
    else:
        f.seek(0)
        pickle.dump(rss, f)
    portalocker.unlock(f)
    f.close()

    # .rss requests
    if request.extension == "rss":
        # return new rss feed xml
        response.headers['Content-Type']='application/rss+xml'
        return rss2.dumps(rss)

    # else send the rss object to be processed by
    # the view
    
    return response.render(dict(rss = rss, rss2 = rss2))