def load_data():
  global banned_users
  global badsubs #banned subs?
  global root_only_subs
  global summon_only_subs
  global imgur_client_id
  global banned_users_page
  global badsubs_page
  global root_only_subs_page
  global summon_only_subs_page
  imgur_client_id = datafile_lines[2].strip()

  #load wiki pages for lists of users and sub settings
  #TODO change sub name
  banned_users_page = r.get_wiki_page('autowikibot','userblacklist')
  badsubs_page = r.get_wiki_page('autowikibot','excludedsubs')
  root_only_subs_page = r.get_wiki_page('autowikibot','rootonlysubs')
  summon_only_subs_page = r.get_wiki_page('autowikibot','summononlysubs')

  #extract info from sub wiki pages
  try:
    banned_users = banned_users_page.content_md.strip().split()
    badsubs = badsubs_page.content_md.strip().split()
    root_only_subs = root_only_subs_page.content_md.strip().split()
    summon_only_subs = summon_only_subs_page.content_md.strip().split()
    success("DATA LOADED")
  except Exception as e:
    #traceback.print_exc()
    fail("DATA LOAD FAILED: %s"%e)
    exit()

def path_from_im(x, output_dir, img_encoding = '.png'):
  if type(x) == type((1,)):
    x = list(x)
    assert x[0] in ('img', 'img_mv')
    mv_im = (x[0] == 'img_mv')
    if type(x[1]) == type(''):
      x[1] = os.path.abspath(x[1])
    im = x[1]
  else:
    mv_im = False
    im = x
  if type(im) == type(''):
    if mv_im:
      new_path = ut.make_temp(os.path.splitext(im)[1], dir = output_dir)
      #new_path = ut.make_temp(img_encoding, dir = output_dir)
      #os.rename(im, new_path)
      ut.sys_check_silent('mv', im, new_path)
      return os.path.split(new_path)[1]
    else:
      return im
  elif type(im) == type(np.array([])):
    path = ut.make_temp(img_encoding, dir = output_dir)
    ig.save(path, im)
    # use relative path in webpage so the directory can be moved
    return os.path.split(path)[1]
  else:
    ut.fail("Don't know how to handle image type: %s" % str(type(im)))

def load_data():
  global banned_users
  global badsubs
  global root_only_subs
  global totalposted
  global imgur_client_id
  global banned_users_comment
  global badsubs_comment
  global root_only_subs_comment
  global totalposted_comment
  imgur_client_id = datafile_lines[2].strip()
  banned_users_comment = "t1_"+datafile_lines[3].strip()
  badsubs_comment = "t1_"+datafile_lines[4].strip()
  root_only_subs_comment = "t1_"+datafile_lines[5].strip()
  totalposted_comment = "t1_"+datafile_lines[6].strip()
  try:
    banned_users = r.get_info(thing_id=banned_users_comment).body.split()
    badsubs = r.get_info(thing_id=badsubs_comment).body.split()
    root_only_subs = r.get_info(thing_id=root_only_subs_comment).body.split()
    totalposted = int(float(r.get_info(thing_id=totalposted_comment).body))
    success("DATA LOADED")
  except Exception as e:
    traceback.print_exc()
    fail("DATA LOAD FAILED: %s"%e)
    exit()

def buildTerminationSignal(check_config):
  """ Builds a termination signal and returns it. """
  kind = check_config.kind
  if not kind in TERMINATION_SIGNALS:
    fail('Unknown termination signal kind: ' + kind)

  return TERMINATION_SIGNALS[kind](check_config)

def find_collector_group_id(client, collector_group_name):
    logging.debug('finding collector group ' + str(collector_group_name))

    # if the root group is set, no need to search
    if collector_group_name == '/':
        return 1

    # trim leading / if it exists
    collector_group_name = collector_group_name.lstrip('/')

    collector_groups = None
    try:
        collector_groups = client.get_collector_group_list(size=-1)
    except ApiException as e:
        err = (
            'Exception when calling get_collector_group_list: ' + str(e) + '\n'
        )
        util.fail(err)

    if collector_groups.status != 200:
        err = (
            'Error ' + str(collector_groups.status) +
            ' calling get_collector_group_list: ' +
            str(collector_groups.errmsg) + '\n'
        )
        util.fail(err)

    # look for matching collector group
    for item in collector_groups.data.items:
        if item.name == collector_group_name:
            return item.id
    return None

def copy_data(data_file, table, mycursor):
    """
    Copy the data contents of DATA_FILE to temporary table TABLE.
    Returns TRUE if successful, FALSE otherwise. Uses cursor MYCURSOR.
    """
    sql = """
        COPY %s
        (
            time_stamp_utc,
            reading
        ) FROM '%s'
        WITH DELIMITER ','
        NULL AS 'Null'
        CSV HEADER
    """ % (table, data_file)

    print("Copying data to temporary table '%s' ..." % (table)),
    
    try:
        exec_and_commit(mycursor, sql)
        util.done()
        return True
    except pyodbc.Error, copy_err:
        util.fail()
        print(copy_err)
        return False

def buildHealthCheck(check_config):
  """ Builds a health check to run and returns it. """
  kind = check_config.kind
  if not kind in HEALTH_CHECKS:
    fail('Unknown health check: ' + kind)
  
  return HEALTH_CHECKS[kind](check_config)

def get_reading_type(meter_id, quantity_id, start_date, end_date, my_cursor):
    """
    Return the reading type for meter METER_ID with quantity id QUANTITY_ID
    between START_DATE and END_DATE (Totalization, Interval) or NONE if an error
    occurs. Uses cursor MY_CURSOR.
    """
    monotonic_sql = """
        SELECT
            CASE WHEN COUNT(*) = 0 THEN 1 ELSE 0 END
            AS IsMonotone
        FROM (
            SELECT TOP 100 ROW_NUMBER() OVER (ORDER BY TimestampUTC)
            AS RowNum, Value
            FROM DataLog2
            WHERE SourceID = %d AND QuantityID = %d
            AND TimestampUTC >= CAST('%s' AS datetime2)
            AND TimestampUTC < CAST('%s' AS datetime2)
        ) T1 INNER JOIN (
            SELECT TOP 100 ROW_NUMBER() OVER (ORDER BY Value)
            AS RowNum, Value
            FROM DataLog2
            WHERE SourceID = %d AND QuantityID = %d
            AND TimestampUTC >= CAST('%s' AS datetime2)
            AND TimestampUTC < CAST('%s' AS datetime2)
        ) T2 ON T1.RowNum = T2.RowNum
            WHERE T1.Value <> T2.Value
    """ % (meter_id, quantity_id, start_date, end_date,
            meter_id, quantity_id, start_date, end_date)
    print("Getting reading type ..."),
    try:
        my_cursor.execute(monotonic_sql)
    except pyodbc.Error, monotonic_err:
        util.fail()
        print(monotonic_err)
        return None

def post_reply(reply,post):
  global badsubs
  global submissioncount
  global totalposted
  try:
    #TODO change name
    #possibly remove? not gonna be nsfw
    reply = "#####	\n\n######	\n\n####	\n"+reply+"^Parent ^commenter ^can [^toggle ^NSFW](/message/compose?to=autowikibot&subject=AutoWikibot NSFW toggle&message=%2Btoggle-nsfw+____id____) ^or[](#or) [^delete](/message/compose?to=autowikibot&subject=AutoWikibot Deletion&message=%2Bdelete+____id____)^. ^Will ^also ^delete ^on ^comment ^score ^of ^-1 ^or ^less. ^| [^(FAQs)](http://www.np.reddit.com/r/autowikibot/wiki/index) ^| [^Mods](http://www.np.reddit.com/r/autowikibot/comments/1x013o/for_moderators_switches_commands_and_css/) ^| [^Magic ^Words](http://www.np.reddit.com/r/autowikibot/comments/1ux484/ask_wikibot/)"
    a = post.reply('[#placeholder-awb]Comment is being processed... It will be automatically replaced by new text within a minute or will be deleted if that fails.')
    postsuccess = r.get_info(thing_id='t1_'+str(a.id)).edit(reply.replace('____id____',str(a.id)))
    if not postsuccess:
      raise Exception ('reply unsuccessful')
    totalposted = totalposted + 1
    submissioncount[str(post.submission.id)]+=1
    success("[OK] #%s "%totalposted)
    return True
  except Exception as e:
    warn("REPLY FAILED: %s @ %s"%(e,post.subreddit))
    if str(e).find('TOO_LONG') > -1:
      a.delete()
    elif str(e) == '403 Client Error: Forbidden' and str(post.subreddit) not in badsubs:
      badsubs = badsubs_page.content_md.strip().split()
      badsubs.append(str(post.subreddit))
      editsummary = 'added '+str(post.subreddit)
      save_changing_variables(editsummary)
    else:
      fail(e)
      a.delete()
    return False

def find_collector_by_description(client, params):
    if 'description' not in params or not params['description']:
        return None

    logging.debug('finding collector ' + str(params['description']))

    collectors = None
    try:
        collectors = client.get_collector_list(size=-1)
    except ApiException as e:
        err = 'Exception when calling get_collector_list: ' + str(e) + '\n'
        util.fail(err)

    if collectors.status != 200:
        err = (
            'Error ' + str(collectors.status) +
            ' calling get_collector_list: ' +
            str(collectors.errmsg) + '\n'
        )
        util.fail(err)

    if 'description' in params and params['description']:
        for item in collectors.data.items:
            if item.description == params['description']:
                return item
    return None

def load_data():
  global banned_users
  global badsubs
  global root_only_subs
  global summon_only_subs
  global imgur_client_id
  global banned_users_page
  global badsubs_page
  global root_only_subs_page
  global summon_only_subs_page
  imgur_client_id = datafile_lines[2].strip()
  banned_users_page = r.get_wiki_page('autowikiabot','userblacklist')
  badsubs_page = r.get_wiki_page('autowikiabot','excludedsubs')
  root_only_subs_page = r.get_wiki_page('autowikiabot','rootonlysubs')
  summon_only_subs_page = r.get_wiki_page('autowikiabot','summononlysubs')
  try:
    banned_users = banned_users_page.content_md.strip().split()
    badsubs = badsubs_page.content_md.strip().split()
    root_only_subs = root_only_subs_page.content_md.strip().split()
    summon_only_subs = summon_only_subs_page.content_md.strip().split()
    success("DATA LOADED")
  except Exception as e:
    #traceback.print_exc()
    fail("DATA LOAD FAILED: %s"%e)
    exit()

def post_reply(reply,post):
  global badsubs
  global submissioncount
  global totalposted
  # This is a quick hack to fix the double list issue (git issue #12)
  # Please find the actual source of this bug, and delete this hack
  # It removes any sentences that are posted more than once
  lines = []
  for line in reply.split("\n"):
    if line not in lines:
      lines.append(line)
  reply = '\n'.join(lines)
  try:
    reply = "#####	\n\n######	\n\n####	\n"+reply+"\n^Parent ^commenter ^can [^toggle ^NSFW](http://www.np.reddit.com/message/compose?to=autowikiabot&subject=AutoWikibot NSFW toggle&message=%2Btoggle-nsfw+____id____) ^or[](#or) [^delete](http://www.np.reddit.com/message/compose?to=autowikiabot&subject=AutoWikibot Deletion&message=%2Bdelete+____id____)^. ^Will ^also ^delete ^on ^comment ^score ^of ^-1 ^or ^less. ^| [^(FAQs)](http://www.np.reddit.com/r/autowikiabot/wiki/index) ^|  [^Source](https://github.com/Timidger/autowikiabot-py)\n ^(Please note this bot is in testing. Any help would be greatly appreciated, even if it is just a bug report! Please checkout the) [^source ^code](https://github.com/Timidger/autowikiabot-py) ^(to submit bugs)"
    a = post.reply('[#placeholder-awb]Comment is being processed... It will be automatically replaced by new text within a minute or will be deleted if that fails.')
    postsuccess = r.get_info(thing_id='t1_'+str(a.id)).edit(reply.replace('____id____',str(a.id)))
    if not postsuccess:
      raise Exception ('reply unsuccessful')
    totalposted = totalposted + 1
    submissioncount[str(post.submission.id)]+=1
    success("[OK] #%s "%totalposted)
    return True
  except Exception as e:
    warn("REPLY FAILED: %s @ %s"%(e,post.subreddit))
    if str(e) == '(TOO_LONG) `this is too long (max: 15000.0)` on field `text`':
      a.delete()
    elif str(e) == '403 Client Error: Forbidden' and str(post.subreddit) not in badsubs:
      badsubs = badsubs_page.content_md.strip().split()
      badsubs.append(str(post.subreddit))
      editsummary = 'added '+str(post.subreddit)
      save_changing_variables(editsummary)
    else:
      fail(e)
      a.delete()
    return False

def color_from_string(s):
  """ todo: add more, see matplotlib.colors.cnames """
  colors = {'r' : (255, 0, 0), 'g' : (0, 255, 0), 'b' : (0, 0, 255)}
  if s in colors:
    return colors[s]
  else:
    ut.fail('unknown color: %s' % s)

def install_collector(client, collector, params):
    fail = False
    installer = download_installer(client, collector, params)

    # ensure installer is executable
    os.chmod(installer, 0755)

    logging.debug('installing ' + str(installer))
    result = util.shell([str(installer), ' -y'])

    if result['code'] != 0 or result['stderr'] != '':
        err = result['stderr']
        # if we failed but there's no stderr, set err msg to stdout
        if err == '':
            err = result['stdout']

        logging.debug('Collector install failed')
        logging.debug('stdout: ' + str(result['stdout']))
        logging.debug('stderr: ' + str(result['stderr']))
        logging.debug('Cleaning up collector install directory')
        util.remove_path(config.INSTALL_PATH + config.AGENT_DIRECTORY)
        fail = True

    # be nice and clean up
    logging.debug('Cleaning up downloaded installer')
    util.remove_path(installer)

    if fail:
        util.fail(err)

def load_ids(id_list, mycursor):
    """
    Insert ID list ID_LIST into the 'meter' table and return the meter ID
    created by the insertion. Returns -1 in case of failure.
    The order of ID_LIST is as follows:
    [ description, unit_id, commodity_id, source_system_id, reading_type_id ]
    Uses cursor MYCURSOR.
    """
    sql = """
        INSERT INTO meter
        (
            description,
            unit_id,
            commodity_id,
            source_system_id,
            reading_type_id
        )
        VALUES      ('%s', %d, %d, %d, %d)
        RETURNING   id
    """ % (id_list[0], id_list[1], id_list[2], id_list[3], id_list[4])

    print("Inserting ID's into 'meter' table ..."),
    
    try:
        exec_and_commit(mycursor, sql)
        result = mycursor.fetchone()
        util.done()
        return result.id
    except pyodbc.Error, get_meter_id_err:
        util.fail()
        print(get_meter_id_err)
        return -1

def go(driver, lsg):
    
    while True:
        try:
            temp = lsg.get()
            if temp == ins.state.go_forward:
                print("Forward")
                driver.tankDrive(0.5, 0.5)
            elif temp == ins.state.go_left:
                print("Left")
                driver.tankDrive(-0.5, 0.5)
            elif temp == ins.state.go_right:
                print("Right")
                driver.tankDrive(0.5, -0.5)
            elif temp == ins.state.wtf:
                print("WTF?")
                driver.tankDrive(0,0)
                util.fail()
            elif temp == ins.state.stop:
                print("Stop")
                driver.tankDrive(0,0)
                break
            elif temp == ins.state.split_detected:
                driver.tankDrive(0,0)
                print("Split")
                
            
        except:
            util.fail()
            break

def file_warning():
  fail("One or more of data files is not found or is corrupted.")
  log("Have them configured as follows:")
  log("already_done_dump - Create empty file if running for first time.")
  log("totalposted - Create empty file if running for first time.")
  log("badsubs - Create empty file if running for first time. Add excluded subreddits if you are using \"all\" as allowed subreddit. Leave empty if allowing specific subreddits.")
  log("banned_users - Create empty file if running for first time. Bot will add banned users automatically. Add manually on separate lines.")
  log("imgur_client_id - Put your imgur client_id in that file")

def test_bb_static():
    for BB in [MyBlackBox3, MyBlackBox4, MyBlackBox5, MyBlackBox6, MyBlackBox7]:
        try:
            mm = BB("MaMaMa")
            fail()
        except ecto.BlackBoxError as e:
            print "Good:"
            print str(e)

 def do_fail():
     debug = False
     try:
         fail('Error: ' + str(e).encode('ascii', errors='replace').decode('ascii'))
     except SystemExit:
         pass
     except AssertionError:
         debug = True
     return debug

def test_modules_spec():
    g = ecto_test.Generate(start=0, step=2)
    x = g["out"]
    x = g["out","out"]
    try:
        x = g[2.0]
        util.fail()
    except TypeError, e:
        print e