def cards_to_database(database, cards_file):
    """
    put data in cards_file into database.
    """
    
    mapper = Mapper(configuration.map_file)
    conn = sqlite3.connect(database)
    cursor = conn.cursor()
    
    cursor.execute("DROP TABLE IF EXISTS cards")
    cursor.execute("""
                   CREATE TABLE IF NOT EXISTS cards
                  (posid TEXT,
                   time INTEGER,
                   statid TEXT)
                   """)
    cursor.execute("CREATE INDEX time_index ON cards(time)")
    cursor.execute("CREATE INDEX statid_index ON cards(statid)")
    cursor.execute("CREATE INDEX posid_index ON cards(posid)")
    with open(cards_file, 'r') as reader:
        for line in reader:
            parts = line.strip().split(',')
            assert len(parts) == 15
            if not mapper.has_statid(parts[9]):
                continue
            if parts[5].count(':') == 1:
                parts[5] = parts[5] + ":00"
            parts[5] = datetime.strptime(parts[5], "%Y/%m/%d %H:%M:%S")
            parts[5] = calendar.timegm(parts[5].utctimetuple())
            cursor.execute("INSERT INTO cards VALUES (?, ?, ?)",
                           (parts[3], parts[5], parts[9]))
            
    cursor.close()
    conn.commit()
    conn.close()

class MapperTests(unittest.TestCase):
        
    def setUp(self):
        self.mapper = Mapper();
        self.utils = Utils();
        utils_for_tests = UtilsForTests();
        self.test_map_scale_to_white_keys = utils_for_tests.loadTestCorpus('test_corpus/test_to_white_keys_corpus');
        self.test_get_map = utils_for_tests.loadTestCorpus('test_corpus/test_get_map_corpus');
    
                
    def test_mapScaleToWhiteKeys(self):
        for case in self.test_map_scale_to_white_keys:
            mapped_scale = self.mapper.mapScaleToWhiteKeys(case[0]);
            self.assertDictEqual(mapped_scale, case[1]);
    
    def test_getMap(self):
        for case in self.test_get_map:
            map = self.mapper.getMap(case[0],case[1]);
            self.assertDictEqual(map, case[2]);
            
    @unittest.skip("Preformance test")        
    def test_TimeitGetMap(self):
        setup = "from utils import Utils; from mapper import Mapper; mapper = Mapper(); utils = Utils();"
        code_to_test = """
        for scale in utils.getAvailableScales():
            for note in utils.getNotes():
                mapper.getMap(note, scale);
        """
        result_first = timeit.repeat(code_to_test, setup = setup,repeat=100, number=100);
        result_avg = reduce(lambda x, y: x + y, result_first) / len(result_first)
        print("Result avg: " + str(result_avg));

def start_thememapper():
    global nav
    global mapper
    #initialize the necessary classes
    mapper = Mapper(get_settings())
    nav = Navigation()
    # Adds the ability to set config file and port through commandline
    p = optparse.OptionParser()
    p.add_option('--port', '-p', default=mapper.port,help='port thememapper should run at')
    p.add_option('--diazo', '-d', default=False,action="store_true",dest="diazo",help='force diazo server to run')
    p.add_option('--diazo_port', '-f', default=mapper.diazo_port,help='port diazo should run at')
    options = p.parse_args()[0]
    mapper.port = options.port
    mapper.diazo_port = options.diazo_port
    #start thememapper
    print "Starting thememapper on http://0.0.0.0:" + mapper.port
    HTTPServer(WSGIContainer(app)).listen(mapper.port)
    if options.diazo or mapper.diazo_run == 'True':
        try: 
            from thememapper.diazo import server
            print "Starting diazo on http://0.0.0.0:" + mapper.diazo_port
            HTTPServer(server.get_application(mapper)).listen(mapper.diazo_port)
        except ImportError: 
            print "You will need to install thememapper.diazo before being able to use this function." 
    ioloop = IOLoop.instance()
    autoreload.watch(os.path.join(os.path.dirname(__file__), 'settings.properties'))
    autoreload.add_reload_hook(reload)
    autoreload.start(ioloop)
    ioloop.start()

def load_source(source, action='load'):


    if source != None:
        db.session.query(Release).filter(Release.source_id == source.id).delete() 

    error_hash = {}
    mapper = Mapper(source)
    for release, error in  mapper.to_ocds():
        if error != None:
            if error[0] not in error_hash:
                error_hash[error[0]] = []
            error_hash[error[0]].append(error[1])
        else:
            load_ocds(release, type='dict', source=source)


    if len(error_hash) > 0:
        message = "Erreurs lors du chargement du fichier %s \n\n" % (source.url)
        message += "\n".join(["Erreur: %s pour les lignes: %s" % (error_type, lines) for error_type, lines in error_hash.items()])
        app.logger.error(message)
    else: 
        app.logger.info("Succès du chargement du fichier : %s" % (source.url))

    source.last_retrieve = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    db.session.commit()

 def __init__(self, dbName, dictClasses=None):
     Mapper.__init__(self, dictClasses)
     self.__initObjDict()
     self.__initUpdateDict()
     try:
         self.db = SqliteObjectsDb(dbName)
     except Exception, ex:
         raise Exception("Error creating SqliteMapper, dbName: %s\n error: %s" % (dbName, ex))

 def test_given_an_image_with_no_points_a_point_map_returned(self):
     img = cv2.imread(os.path.join(self.test_data_path,'SimpleTestImage5.png'),1)
     expected = [-1 for i in range(0,20)]
     colour = [255,255,255]
     threshold = 0
     mapper = Mapper(colour,threshold)
     actual = mapper.get_points(img)
     self.assertEquals(expected,actual)

 def test_given_an_colour_image_and_specific_colour_a_point_map_returned(self):
     img = cv2.imread(os.path.join(self.test_data_path,'SimpleTestImage2.png'),1)
     expected = [i for i in range(0,20)]
     colour = [255,128,0]
     threshold = 0
     mapper = Mapper(colour,threshold)
     actual = mapper.get_points(img)
     self.assertEquals(expected,actual)

 def test_given_a_threshold_items_in_threshold_work_for_blue(self):
     img = cv2.imread(os.path.join(self.test_data_path,'BlueThresholdTest.png'),1)
     threshold = 20
     expected = [0,0,0,-1,-1] 
     colour = [128,128,128]
     mapper = Mapper(colour, threshold)
     actual = mapper.get_points(img)
     self.assertEquals(expected,actual)

 def test_a_threshold_can_be_changed(self):
     img = cv2.imread(os.path.join(self.test_data_path,'GreenThresholdTest.png'),1)
     initial_threshold = 20
     new_threshold = 21
     expected = [0,0,0,0,0] 
     colour = [128,128,128]
     mapper = Mapper(colour, initial_threshold)
     mapper.set_threshold(new_threshold)
     actual = mapper.get_points(img)
     self.assertEquals(expected,actual)

class TestMapper(object):
    used = False

    def empty(self, *args, **kw):
        self.used = True

    def setUp(self):
        self.used = False
        self.opts = Opts()
        self.mapper = Mapper(self.opts)
        self.mapper.save_config = self.empty

    def parse_opts_test(self):
        self.opts.title = 'название 1'.decode('utf-8')
        self.mapper.prepare()
        assert (self.mapper.config['title'] == u'название 1')
        assert self.used

    def load_config_test(self):
        config = {'title': 'название 2'}
        f = NamedTemporaryFile()
        json.dump(config, f, encoding='utf-8')
        f.flush()
        self.opts.config = f.name
        self.mapper.prepare()
        assert (self.mapper.config['title'] == u'название 2')

    def get_pagesize_test(self):
        assert self.mapper.get_page_size('a4') == self.mapper.get_page_size('A4')
        x, y = self.mapper.get_page_size('a4')
        assert self.mapper.get_page_size('a4', True) == (y, x)
        assert self.mapper.get_page_size('a3', False) == (y, x * 2)

    def get_coords_test(self):
        assert self.mapper.get_coords('235,117.2') == [235., 117.2]

 def __init__(self, dbName, dictClasses=None, tablePrefix=''):
     Mapper.__init__(self, dictClasses)
     self._objTemplate = None
     try:
         self.db = SqliteFlatDb(dbName, tablePrefix)
         self.doCreateTables = self.db.missingTables()
         
         if not self.doCreateTables:
             self.__loadObjDict()
     except Exception, ex:
         raise Exception('Error creating SqliteFlatMapper, dbName: %s, tablePrefix: %s\n error: %s' % (dbName, tablePrefix, ex))

class Store(object):
    """Store.
       Required constructor parameters: path, block_size, hash_algorithm,
       umask, blockpool, mappool.
    """

    def __init__(self, **params):
        umask = params['umask']
        if umask is not None:
            os.umask(umask)

        path = params['path']
        if path and not os.path.exists(path):
            os.makedirs(path)
        if not os.path.isdir(path):
            raise RuntimeError("Cannot open path '%s'" % (path,))

        p = {'blocksize': params['block_size'],
             'blockpath': os.path.join(path + '/blocks'),
             'hashtype': params['hash_algorithm'],
             'blockpool': params['blockpool']}
        self.blocker = Blocker(**p)
        p = {'mappath': os.path.join(path + '/maps'),
             'namelen': self.blocker.hashlen,
             'mappool': params['mappool']}
        self.mapper = Mapper(**p)

    def map_get(self, name):
        return self.mapper.map_retr(name)

    def map_put(self, name, map):
        self.mapper.map_stor(name, map)

    def map_delete(self, name):
        pass

    def block_get(self, hash):
        blocks = self.blocker.block_retr((hash,))
        if not blocks:
            return None
        return blocks[0]

    def block_put(self, data):
        hashes, absent = self.blocker.block_stor((data,))
        return hashes[0]

    def block_update(self, hash, offset, data):
        h, e = self.blocker.block_delta(hash, offset, data)
        return h

    def block_search(self, map):
        return self.blocker.block_ping(map)

class Store(object):
    """Store.
       Required constructor parameters: path, block_size, hash_algorithm,
       blockpool, mappool.
    """

    def __init__(self, **params):
        pb = {'blocksize': params['block_size'],
              'hashtype': params['hash_algorithm'],
              'archipelago_cfile': params['archipelago_cfile'],
              }
        self.blocker = Blocker(**pb)
        pm = {'namelen': self.blocker.hashlen,
              'hashtype': params['hash_algorithm'],
              'archipelago_cfile': params['archipelago_cfile'],
              }
        self.mapper = Mapper(**pm)

    def map_get(self, name, size):
        return self.mapper.map_retr(name, size)

    def map_put(self, name, map, size, block_size):
        self.mapper.map_stor(name, map, size, block_size)

    def map_delete(self, name):
        pass

    def map_copy(self, dst, src, size):
        self.mapper.map_copy(dst, src, size)

    def block_get(self, hash):
        blocks = self.blocker.block_retr((hash,))
        if not blocks:
            return None
        return blocks[0]

    def block_get_archipelago(self, hash):
        blocks = self.blocker.block_retr_archipelago((hash,))
        if not blocks:
            return None
        return blocks[0]

    def block_put(self, data):
        hashes, absent = self.blocker.block_stor((data,))
        return hashes[0]

    def block_update(self, hash, offset, data):
        h, e = self.blocker.block_delta(hash, offset, data)
        return h

    def block_search(self, map):
        return self.blocker.block_ping(map)

    def test_a_colour_can_be_changed(self):
        img = cv2.imread(os.path.join(self.test_data_path,'GreenThresholdTest.png'),1)
        threshold = 20
        initial_expected = [0,0,0,-1,-1] 
        initial_colour = [128,128,128]
        new_expected = [-1,-1,-1,-1,-1]
        new_colour = [64,64,64]
        mapper = Mapper(initial_colour, threshold)
        initial_result = mapper.get_points(img)
        self.assertEquals(initial_expected,initial_result)

        mapper.set_colour(new_colour)
        new_result = mapper.get_points(img)
        self.assertEquals(new_expected,new_result)

 def __init__(self, lcg, map_size):
     self.current_player_index = None
     self.game_started = False
     self._players = []
     self.lcg = lcg
     self.handlers = Handlers(self)
     self.mapper = Mapper(self, *map_size)

 def __init__(self, connection_config):
     self.connection_config = connection_config
     provider_name = self.connection_config['cloud_provider_name']
     provider_name = transfer_cloud_provider_name(provider_name)
     from mapper import Mapper
     self.mapper = Mapper(provider_name)
     self.driver = self.mapper.connect(self.connection_config)

    def __init__(self, outputRepo, inputRepos=None):
        """Construct a Butler to manage an output (read/write) repository,
        attaching zero or more input (read-only) repositories."""

        self.mapper = Mapper.create(outputRepo, inputRepos)
        self.registryPath = self.mapper.registryPath
        self.provenance = []

def map_result():
    
    if request.method == 'POST':
        rdf = request.form['data']
        try:
            gg = Graph()
            rdf_content = StringIO.StringIO(rdf.encode('utf-8'))
            gg.parse(rdf_content,  format="xml")
            mapper = Mapper(gg)
            #TODO-remove this test block after release
            batch_json = open("test_mapper_batch.json","r").read()
            result = mapper.mapping(batch_json)
            #End TODO-remove this test block after release
            return Response(result, mimetype='application/rdf+xml') 
        except:
            pass

def job(grouperNum, chunksQueue, listSaveStateNameGrouper, listListLastCallNum):
    print 'Starting worker ' + str(grouperNum)      
    while True:
        # Get new chunck to process
        chunk = chunksQueue.get()         
        # Work
        print 'Worker ' + str(grouperNum) + ' mapping chunk ' + str(chunk)
        MapIterator = MapChunkIterator(mapChunksNameGenerator(chunk)) # Iterator to iterate through the chunck        
        theContext = MapContext(groupChunksNameGenerator(chunk),MapIterator)        
        Mapper.map(theContext)
        print 'Worker ' + str(grouperNum) + ' grouping locally chunck ' + str(chunk)        
        idx = listListLastCallNum[grouperNum]+1        
        theGrouper = Grouper(grouperNum,idx,idx-1,directory);        
        listSaveStateNameGrouper[grouperNum] = theGrouper.group(theContext)
        listListLastCallNum[grouperNum] = idx ;      
        # "Close" chunk
        chunksQueue.task_done()

 def __init__(self):
     #may want to enhance this with a pre-load file to prepopulate the DB
     self.db = {}                   # dictionary of DNSClassifierEntrys
     self.mapper = Mapper()
     self.new_callbacks = []        # For each new entry
     self.update_callbacks = []     # For each time an entry is updated
     self.all_callbacks = []        # When entry is updated or new
     self.class_callbacks = {}      # Dictionary of lists of callbacks per