def get_comparison(news_org, NP_to_sentence, VP_to_sentence,
                   NPs, VPs, NP_synsets, VP_synsets,
                   article_topic, article_headline, article_news_org):
  '''Compares the articles from a single NewsOrg to an article that is
  represented by its NPs and VPs.'''
  # synsets aren't picklable so they're stored as (pos, offset) and unpacked
  NP_synsets = [wn._synset_from_pos_and_offset(pos, offset)
                for (pos, offset) in NP_synsets]
  VP_synsets = [wn._synset_from_pos_and_offset(pos, offset)
                for (pos, offset) in VP_synsets]

  comparison_articles = news_org.get_query_results(article_topic)
  if not comparison_articles:
    logger.log.warning("No comparison articles for %s" % news_org)
    return []
  comparisons = []
  for comparison_article in comparison_articles:
    if (news_org == article_news_org and
        comparison_article.headline == article_headline):
      # comparison_article is likely the same as the original article
      # do not compare
      pass
    try:
      comparison = compare_articles.compare_articles(NP_to_sentence,
                                                     VP_to_sentence, NPs, VPs,
                                                     NP_synsets, VP_synsets,
                                                     comparison_article)
      if comparison:
        comparisons.append(comparison)
    except:
      continue
  return comparisons

def get_comparison(news_org, article_topic, NP_to_sentence, VP_to_sentence,
                   NPs, VPs, NP_synsets, VP_synsets, article):
  '''Compares the articles from a single NewsOrg to an article that is
  represented by its NPs and VPs.'''
  # synsets aren't picklable so they're stored as (pos, offset) and unpacked
  NP_synsets = [wn._synset_from_pos_and_offset(pos, offset)
                for (pos, offset) in NP_synsets]
  VP_synsets = [wn._synset_from_pos_and_offset(pos, offset)
                for (pos, offset) in VP_synsets]

  comparison_articles = news_org.get_query_results(article_topic)
  if not comparison_articles:
    return []
  comparisons = []
  for comparison_article in comparison_articles:
    try:
      comparison = compare_articles.compare_articles(NP_to_sentence,
                                                     VP_to_sentence, NPs, VPs,
                                                     NP_synsets, VP_synsets,
                                                     comparison_article)
      if comparison:
        comparisons.append(comparison)
    except:
      continue
  return comparisons

def loadAll():
    if loaded:
        return
    index = wn._lemma_pos_offset_map
    print 'loading wordnet into cache... '
    cache = wn._synset_offset_cache
    f = open('pos_offset.txt','r')
    for line in f:
        ll = line.split()
        pos = ll[0]
        offset = int(ll[1])
        wn._synset_from_pos_and_offset(pos,offset)
    print 'Done: '+str(sum([len(cache[x]) for x in cache]))+'/'+str(len(index))
    loaded = True

 def random_le_and_sy(self):
     '''
     '''
     from nltk.corpus import wordnet as wn
     
     start_at = random.choice( range( len(self.orbn_ids)))
     
     for counter,le_obj in enumerate(self.les_get_generator()):
         
         if counter >= start_at:
             print()
             print(etree.tostring(le_obj.le_el,
                                  pretty_print=True))
             answer = input('interesting? ')
             if answer == 'y':
                 target = le_obj.get_synset_id()
                 eng,version,offset,pos = target.split('-')
                 sy_obj = self.synsets_find_synset(target)
                 print()
                 print(etree.tostring(sy_obj.synset_el,
                                      pretty_print=True))
                 synset = wn._synset_from_pos_and_offset(pos,int(offset))
                 print(synset.lemmas())
                 print(synset.definition())
                 input('continue?')

 def all_senti_synsets(self):
     from nltk.corpus import wordnet as wn
     for key, fields in self._db.items():
         pos, offset = key
         pos_score, neg_score = fields
         synset = wn._synset_from_pos_and_offset(pos, offset)
         yield SentiSynset(pos_score, neg_score, synset)

 def __getitem__(self, k):
     for pos in ("n", "v", "a", "r"):
         try:
             synset = wn._synset_from_pos_and_offset(pos, k)
         except:
             pass
         if synset:
             return synset
     return None

    def synset(self, id, pos=ADJECTIVE):
        if pos in _pattern2wordnet:
            pos = _pattern2wordnet[pos]
        try:
            s = wn._synset_from_pos_and_offset(pos, id)
            lemma = s.lemma_names()[0]
            return self[lemma]
        except:
            pass

        return None

def offset_to_synset(offset):
    """ 
    Look up a synset given offset-pos 
    (Thanks for @FBond, see http://moin.delph-in.net/SemCor)
    >>> synset = offset_to_synset('02614387-v')
    >>> print '%08d-%s' % (synset.offset, synset.pos)
    >>> print synset, synset.definition
    02614387-v
    Synset('live.v.02') lead a certain kind of life; live in a certain style
    """
    return wn._synset_from_pos_and_offset(str(offset[-1:]), int(offset[:8]))

def get_lemma(synset_id):
    try:
        wn_id = synset_id.split(':')[1]
        offset = int(wn_id[:-1])
        pos = wn_id[-1]
        print offset
        print pos
        return wordnet._synset_from_pos_and_offset(pos, offset)
    except:
        log.error("get_lemma(): error looking up synset id {0} in NLTK WordNet".format(synset_id))
        return None

 def synsets_from_txt(self, fname):
     with open(fname, "rb") as f:
         lines = f.readlines()
     df = []
     for line in lines:
         w = line.split()[0]
         descr = line.strip("\r\n").replace(w + " ", "")
         synset = wn._synset_from_pos_and_offset(w[0], int(w[1:]))
         df.append({"id": w, "names": descr, "synset": synset})
     # df = pandas.DataFrame(df, columns=['id', 'names', 'synset'])
     return df

    def synsets_from_csv(self, fname):
        sf = pandas.read_csv(fname, sep='\t')
        df = []
        for idx, row in sf.iterrows():
            idd = row['synset_id']
            try:
                synset = wn._synset_from_pos_and_offset(idd[0], int(idd[1:]))
            except:
                import pdb; pdb.set_trace()

            df.append({'id':idd, 'names':row['name'], 'synset':synset})
        return df

    def pred_acc(self, compute_acc=True):
        if compute_acc:
            preds = self.predict()
        imagenet_labels = self.synsets_from_txt("synset_words.txt")
        dataset_labels = self.synsets_from_csv(os.path.join(self.exp, "data", self.exp + ".csv"))
        all_hyps = lambda s: s.hyponyms()

        df = pandas.DataFrame.from_dict(dataset_labels)
        df["imgid"] = ""
        df["imdnames"] = ""
        df["kind"] = "unknown"
        df["accuracy"] = np.nan
        df["accuracy0"] = np.nan
        df["confidence0"] = np.nan
        for no, dtlab in enumerate(dataset_labels):
            hypos = set([i for i in dtlab["synset"].closure(all_hyps)])
            hypos = hypos.union([dtlab["synset"]])
            for imglab in imagenet_labels:
                if imglab["synset"] in hypos:
                    df.loc[no, "imgid"] = imglab["id"]
                    df.loc[no, "imgnames"] = imglab["names"]
                    if imglab["id"] == df.loc[no, "id"]:
                        df.loc[no, "kind"] = "exact"
                    else:
                        df.loc[no, "kind"] = "superordinate"
                    break
            if compute_acc:
                acc = False
                acc0 = False
                for i, p in enumerate(preds[no]):
                    psyn = wn._synset_from_pos_and_offset(p["synset"][0], int(p["synset"][1:]))
                    df.loc[no, "pred%d" % i] = ", ".join(psyn.lemma_names())
                    # check if the prediction is exact
                    # or at least more specific than the correct resp
                    if psyn in hypos:
                        acc = True
                    if i == 0:
                        if psyn in hypos:
                            acc0 = True
                if acc == False:
                    if df.loc[no, "kind"] != "unknown":
                        df.loc[no, "accuracy"] = False
                else:
                    df.loc[no, "accuracy"] = True
                if acc0 == False:
                    if df.loc[no, "kind"] != "unknown":
                        df.loc[no, "accuracy0"] = False
                else:
                    df.loc[no, "accuracy0"] = True
                df.loc[no, "confidence0"] = preds[no][0]["confidence"]
        return df

    def create_obj_scale_dictionary(obj_names, scales_dic, key_type='synset'):
        """
        Creates a dictionary containing association between objects in the list_aws file and their synset names given
        A dictionary of desired category<>scale or synset<>scale associations.
        Category names can be retrieved from the 'shapenet_synset' field on the shapenet meta db.
        Example of such categories are: 'airplane', 'bag', 'bathtub', 'train'

        :param obj_names: list containing the object names
        :param scales_dic: A dictionary containing scale values for particualr class of objects
        :param key_type: Type of keys in scales_dic. Could be 'synset' (Default) or 'category'.
        :return: association dictionary with object names as keys
        """
        try:
            print('Getting synset offsets from MongoDB...')
            db_client = pm.MongoClient(port=22334)
            table = db_client['synthetic_generative']['3d_models']
            cursor = table.find({'type': 'shapenet',
                                 'version': 2,
                                 'id': {'$in': obj_names}})

            if key_type == 'category':
                obj_scale_dic = dict()  # Stores the table for id-scale correspondence
                for doc in cursor:
                    offset = doc['shapenet_synset']
                    synset_name = wn._synset_from_pos_and_offset(offset[0], int(offset[1:])).name()
                    basic_name = synset_name.split('.')[0]  # Take the first part of the synset name
                    if basic_name in scales_dic:
                        obj_scale_dic["http://threedworld.s3.amazonaws.com/"+doc['id']+'.bundle'] = \
                            {'option': 'Multi_size', 'scale': scales_dic[basic_name]}
                    else:
                        obj_scale_dic["http://threedworld.s3.amazonaws.com/"+doc['id']+'.bundle'] = \
                            {'option': 'Multi_size', 'scale': 1}
            elif key_type == 'synset':
                obj_scale_dic = dict()  # Stores the table for id-scale correspondence
                for doc in cursor:
                    offset = doc['shapenet_synset']
                    if offset in scales_dic:
                        obj_scale_dic["http://threedworld.s3.amazonaws.com/" + doc['id'] + '.bundle'] = \
                            {'option': 'Multi_size', 'scale': scales_dic[offset]}
                    else:
                        obj_scale_dic["http://threedworld.s3.amazonaws.com/" + doc['id'] + '.bundle'] = \
                            {'option': 'Multi_size', 'scale': 1}
            else:
                raise ValueError("Key type should be either: 'synset' or 'category'.")
            print('Table created!')
            db_client.close()
        except:
            print('Could not connect to DB. Create a SSH tunnel.')
            raise

        return obj_scale_dic

 def senti_synset(self, *vals):        
     if tuple(vals) in self.db:
         pos_score, neg_score = self.db[tuple(vals)]
         pos, offset = vals
         synset = wn._synset_from_pos_and_offset(pos, offset)
         return SentiSynset(pos_score, neg_score, synset)
     else:
         synset = wn.synset(vals[0])
         pos = synset.pos
         offset = synset.offset
         if (pos, offset) in self.db:
             pos_score, neg_score = self.db[(pos, offset)]
             return SentiSynset(pos_score, neg_score, synset)
         else:
             return None

    def synsets_from_csv(self, fname, sep=","):
        with open(fname, "rb") as f:
            lines = f.readlines()
        df = []
        for line in lines:
            spl = line.strip("\n").split(sep)
            try:
                synset = wn._synset_from_pos_and_offset(spl[0][0], int(spl[0][1:]))
            except:
                import pdb

                pdb.set_trace()

            df.append({"id": spl[0], "names": spl[1], "synset": synset})
        # df = pandas.DataFrame(df, columns=['id', 'names', 'synset'])
        return df

    def __init__(self, synset):
        """ A set of synonyms that share a common meaning.
        """
        if isinstance(synset, WordNetSynset):
            self._wnsynset = synset
        elif isinstance(synset, Synset):
            self = self
        elif isinstance(synset, (tuple, int)):
            if isinstance(synset, int):
                synset = (synset, "NN")
            offset, pos = synset
            self._wnsynset = wn._synset_from_pos_and_offset(_pattern2wordnet[pos] if pos in _pattern2wordnet else pos, offset)
        else:
            raise NotImplementedError

        self._synset = _synset

 def senti_synset(self, *vals):        
     from nltk.corpus import wordnet as wn
     if tuple(vals) in self._db:
         pos_score, neg_score = self._db[tuple(vals)]
         pos, offset = vals
         synset = wn._synset_from_pos_and_offset(pos, offset)
         return SentiSynset(pos_score, neg_score, synset)
     else:
         synset = wn.synset(vals[0])
         pos = synset.pos()
         offset = synset.offset()
         if (pos, offset) in self._db:
             pos_score, neg_score = self._db[(pos, offset)]
             return SentiSynset(pos_score, neg_score, synset)
         else:
             return None

 def senti_synset(self, *vals):   
     '''
     get pos and neg scores for word
     input: vals is a word or word, pos tag
     output: return an object SentiSynset with word, pos score and neg score set in the object     
     '''
     if tuple(vals) in self.db:
         pos_score, neg_score = self.db[tuple(vals)]
         pos, offset = vals
         synset = wn._synset_from_pos_and_offset(pos, offset)
         return SentiSynset(pos_score, neg_score, synset)
     else:
         synset = wn.synset(vals[0])
         pos = synset.pos
         offset = synset.offset
         if (pos, offset) in self.db:
             pos_score, neg_score = self.db[(pos, offset)]
             return SentiSynset(pos_score, neg_score, synset)
         else:
             return None

	def parseXML(self, file):
		""" 
		form: 
		(file_name) --> print

		description: 
		function that fills the synset2synonym & synset2word dict

		exemple:
		>>> graphWN.parseXML("wolf-1.0b4.xml")
		XML parsed

		"""
		tree = etree.parse(file)
		synsets=tree.xpath("SYNSET")
		synonyms=[]
		for synset in synsets:
			synset_id_orig=synset.findtext("ID")
			synonymsEl=synset.findall("SYNONYM/LITERAL")
			syn=[]
			for synonym in synonymsEl: #obtient une liste des synonymes à partir d'une liste des element SYNONYM
			  syn.append(synonym.text)
			  #if synonym.text in synonyms:
			    #print "IN"
			  #else:
			    #synonyms.append(synonym.text)
			self.synset2synonym[synset_id_orig]=syn
			#print self.synset2synonym
			synset_id, pos = synset_id_orig.split("-")[2], synset_id_orig.split("-")[3]
  			# mapping fr-en b --> r
  			if pos == "b":
  				pos = "r"
  			synset = wn._synset_from_pos_and_offset(pos,int(synset_id))
  			synset_name = synset.name.split(".")[0]
  			if synset_id_orig not in self.synset2word:
  				self.synset2word[synset_id_orig] = synset_name
  		print "XML parsed"

 def all_senti_synsets(self):
     for key, fields in self.db.iteritems():
         pos, offset = key
         pos_score, neg_score = fields
         synset = wn._synset_from_pos_and_offset(pos, offset)
         yield SentiSynset(pos_score, neg_score, synset)