def check_normalized(self, word_vocalised, resulted_data):
        """
		If the entred word is like the found word in dictionary, to treat some normalized cases, 
		the analyzer return the vocalized like words;
		ُIf the word is ذئب, the normalized form is ذءب, which can give from dictionary ذئبـ ذؤب.
		this function filter normalized resulted word according the given word, and give ذئب.
		@param word_vocalised: the input word.
		@type word_vocalised: unicode.
		@param resulted_data: the founded resulat from dictionary.
		@type resulted_data: list of dict.
		@return: list of dictionaries of analyzed words with tags.
		@rtype: list.
		"""
        # print word_vocalised.encode('utf8');
        filtred_data = []
        inputword = araby.stripTashkeel(word_vocalised)
        for item in resulted_data:
            if "vocalized" in item.__dict__:  # .has_key('vocalized') :
                # ~ if 'vocalized' in item :
                # ~ outputword = araby.stripTashkeel(item['vocalized'])
                outputword = araby.stripTashkeel(item.__dict__["vocalized"])
                # print u'\t'.join([inputword, outputword]).encode('utf8');
                if inputword == outputword:
                    # item['tags']+=':a';
                    filtred_data.append(item)
        return filtred_data

	def getStemVariants(self,stem,prefix,suffix):
		"""
		Generate the Noun stem variants according to the affixes.
		For example مدرستي=>مدرست+ي => مدرسة +ي.
		Return a list of possible cases.
		@param stem: the input stem.
		@type stem: unicode.
		@param prefix: prefixe.
		@type prefix: unicode.
		@param suffix: suffixe.
		@type suffix: unicode.
		@return: list of stem variants.
		@rtype: list of unicode.
		"""
		#some cases must have some correction
		#determinate the prefix and suffix types
		# create a list, the first item is the verb without changes
		prefix_possible_noun_list= set([stem])
		# Prefix
		prefix=araby.stripTashkeel(prefix);
		suffix=araby.stripTashkeel(suffix);
		possible_noun_list=prefix_possible_noun_list;
		if suffix in (araby.ALEF+araby.TEH, araby.YEH+araby.TEH_MARBUTA,araby.YEH, araby.YEH+araby.ALEF+araby.TEH):
			possible_noun=stem+araby.TEH_MARBUTA;
			possible_noun_list.add(possible_noun)
		if suffix=="" or suffix==araby.YEH+araby.NOON or suffix==araby.WAW+araby.NOON:
			possible_noun=stem+araby.YEH;
			possible_noun_list.add(possible_noun)
		if stem.endswith(araby.YEH):
			possible_noun=stem[:-1]+araby.ALEF_MAKSURA;
			possible_noun_list.add(possible_noun)
		#to be validated
		validated_list=possible_noun_list;
		return validated_list

    def isPossibleCollocation(self, list2, context="", lenght=2):
        """
		Guess if the given list is a possible collocation
		This is used to collect unkown collocations, from user input
		return True oor false
		@param wordlist: word of list, 2 or more words.
		@type wordlist: list of unicode.
		@param lenght: minimum number of words in the collocation
		@type lenght: integer.		
		@return : the rule of found collocation, 100 default.
		@rtype: interger.
		"""
        if len(list2) < lenght:
            return 0
        else:
            itemV1 = list2[0]
            itemV2 = list2[1]
            item1 = araby.stripTashkeel(itemV1)
            item2 = araby.stripTashkeel(itemV2)
            # if item1[-1:] in (u".",u"?",u",",u'[', u']',u'(',')'):
            # 	return 0;
            if not collocation_const.token_pat.search(item1) or not collocation_const.token_pat.search(item2):
                return -1
                # else: return 100;
            elif item1 in collocation_const.ADDITIONAL_WORDS:
                return 10
            elif item1 in collocation_const.NAMED_PRIOR:
                return 15
            elif item2 not in collocation_const.SPECIAL_DEFINED:
                if item2.startswith(u"ال") and item1.startswith(
                    u"ال"
                ):  # re.search(ur'^(ال|بال|وبال|فال|وال|لل|كال|فكال|ولل|فلل|فبال)', item1):
                    return 20
                elif item1.endswith(u"ة") and item2.startswith(u"ال"):
                    return 30

                    # حالة الكلمات التي تبدأ بلام الجر والتعريف
                    # لا داعي لها لأنها دائما مجرورة
                    # if  item2.startswith(u'لل'):
                    # 	return 40;
                elif item1.endswith(u"ة") and item2.endswith(u"ة"):
                    return 40
                    # if item1.endswith(u'ي') and item2.endswith(u'ي'):
                    # 	return 60;

                elif context != u"" and context in collocation_const.tab_noun_context and item2.startswith(u"ال"):
                    return 50
                    # return True;

                elif item1.endswith(u"ات") and item2.startswith(u"ال"):
                    return 60
            return 100

	def getSuffixVariant(self, word, suffix, enclitic):
		"""
		Get the suffix variant to be joined to the word.
		For example: word = مدرس, suffix=ة, encletic=ي. The suffix is converted to Teh.
		@param word: word found in dictionary.
		@type word: unicode.
		@param suffix: second level suffix.
		@type suffix: unicode.
		@param enclitic: first level suffix.
		@type enclitic: unicode.		
		@return: variant of suffixes  (vocalized suffix and vocalized suffix without I'rab short mark).
		@rtype: (unicode, unicode)
		"""
		enclitic_nm=araby.stripTashkeel(enclitic)
		newSuffix =suffix; #default value
		#if the word ends by a haraka
		if suffix.find(araby.TEH_MARBUTA)>=0 and len (enclitic_nm)>0:
			newSuffix=re.sub(araby.TEH_MARBUTA, araby.TEH, suffix);
		elif 	not enclitic_nm and word[-1:] in (araby.ALEF_MAKSURA, araby.YEH, araby.ALEF) and araby.isHaraka(suffix):
			newSuffix=u"";
		#gererate the suffix without I'rab short mark
		# here we lookup with given suffix because the new suffix is changed and can be not found in table
		if u'متحرك' in stem_noun_const.CONJ_SUFFIX_LIST_TAGS[suffix]['tags']:
			suffixNonIrabMark =araby.stripLastHaraka(newSuffix);
		else:
			suffixNonIrabMark = newSuffix
		return newSuffix, suffixNonIrabMark ;

def Comparetashkeel(text):
	import tashkeel.tashkeel as ArabicVocalizer
	# the entred text is vocalized correctly
	correct_text=text;
	text=araby.stripTashkeel(text);
	vocalizer=ArabicVocalizer.TashkeelClass();
	vocalized_text=vocalizer.tashkeel(text);
	
	# compare voalized text with a correct text
	text1=correct_text;
	text2=vocalized_text;
	# remove collocations symboles
	text2=text2.replace("'","");
	text2=text2.replace("~","");
	
	#stemmer=tashaphyne.stemming.ArabicLightStemmer()
	list1=vocalizer.analyzer.tokenize(text1);
	list2=vocalizer.analyzer.tokenize(text2);
	print u":".join(list1).encode('utf8');
	print u":".join(list2).encode('utf8');
	correct=0;
	incorrect=0;
	total=len(list1);
	if len(list1)!=len(list2):
		print "lists haven't the same length";
	else:
		for i in range(total):
			if araby.vocalizedlike(list1[i],list2[i]):
				correct+=1;
			else:
				incorrect+=1;
	
	result=[vocalized_text,"correct:%0.2f%%"%round(correct*100.00/total,2),"incorrect:%0.2f%%"%round(incorrect*100.00/total,2),total]
	return result#correct*100/total;

	def generate_possible_conjug(self, infinitive_verb, unstemed_verb , affix, future_type=araby.FATHA, externPrefix="-", externSuffix="-", transitive=True):
		"""
		"""
	##    future_type=FATHA;
		#~ transitive=True;
		list_correct_conj=[];
		if infinitive_verb=="" or unstemed_verb=="" or affix=="":
			return set();
		verb = infinitive_verb;
		future_type = libqutrub.ar_verb.get_future_type_entree(future_type);
		#print u"\t".join([verb, future_type]).encode('utf8');
		vb = libqutrub.classverb.verbclass(verb, transitive, future_type);
		# الألف ليست جزءا من السابقة، لأنها تستعمل لمنع الابتداء بساكن
		# وتصريف الفعل في الامر يولده
		if affix.startswith(araby.ALEF): affix=affix[1:]
		# get all tenses to conjugate the verb one time
		tenses=[];
		if stem_verb_const.Table_affix.has_key(affix):
			for pair in stem_verb_const.Table_affix[affix]:
				tenses.append(pair[0]);#tense=pair[0]
		tenses=list(set(tenses)); # avoid duplicata 


		if stem_verb_const.Table_affix.has_key(affix):
			for pair in stem_verb_const.Table_affix[affix]:
				tense=pair[0]
				pronoun=pair[1]
				if self.is_compatible_proaffix_tense(externPrefix, externSuffix, tense, pronoun, transitive):

					conj_vocalized = vb.conjugateTenseForPronoun( tense, pronoun)
					#strip all marks and shadda
					conj_nm =  araby.stripTashkeel(conj_vocalized);
					if conj_nm==unstemed_verb:
						list_correct_conj.append({'verb':infinitive_verb, 'tense':tense, 'pronoun':pronoun, 'vocalized':conj_vocalized, 'unvocalized':conj_nm});
		return list_correct_conj;

	def getWordVariant(self, word, suffix):
		"""
		Get the word variant to be joined to the suffix.
		For example: word = مدرسة, suffix=ي. The word is converted to مدرست.
		@param word: word found in dictionary.
		@type word: unicode.
		@param suffix: suffix ( firts or second level).
		@type suffix: unicode.
		@return: variant of word.
		@rtype: unicode.
		"""
		word_stem=word;
		# print word.encode('utf8');
		#HARAKAT=(FATHA, DAMMA, KASRA, SUKUN, DAMMA, DAMMATAN, KASRATAN, FATHATAN);
		suffix_nm=araby.stripTashkeel(suffix)
		#if the word ends by a haraka
		word_stem=araby.stripLastHaraka(word_stem);

		if word_stem.endswith(araby.TEH_MARBUTA) and suffix_nm in (araby.ALEF+araby.TEH, araby.YEH+araby.TEH_MARBUTA, araby.YEH, araby.YEH+araby.ALEF+araby.TEH):
			word_stem=word_stem[:-1];
		elif word_stem.endswith(araby.TEH_MARBUTA) and suffix_nm!=u"":
			word_stem=word_stem[:-1]+araby.TEH;
		elif word_stem.endswith(araby.ALEF_MAKSURA) and suffix_nm!=u"":
			word_stem = word_stem[:-1]+araby.YEH;			
		elif word_stem.endswith(araby.HAMZA) and suffix_nm!=u"":
			if suffix.startswith(araby.DAMMA):
				word_stem = word_stem[:-1] + araby.WAW_HAMZA;
			elif suffix.startswith(araby.KASRA):
				word_stem = word_stem[:-1] + araby.YEH_HAMZA;
				
		return word_stem;

	def check_word(self,word, guessedTag=""):
		"""
		Analyze one word morphologically as verbs
		@param word: the input word.
		@type word: unicode.
		@return: list of dictionaries of analyzed words with tags.
		@rtype: list.
		"""	
		word=araby.stripTatweel(word);
		word_vocalised=word;
		word_nm=araby.stripTashkeel(word);
		resulted_text=u"";
		resulted_data=[];
		# if word is a pounctuation
		resulted_data+=self.check_word_as_pounct(word_nm);
		# Done: if the word is a stop word we have  some problems,
		# the stop word can also be another normal word (verb or noun),
		# we must consider it in future works
		# if word is stopword allow stop words analysis
		resulted_data+=self.check_word_as_stopword(word_nm);

		#if word is verb
		# مشكلة بعض الكلمات المستبعدة تعتبر أفعلا أو اسماء
		if  self.tagger.hasVerbTag(guessedTag) or self.tagger.isStopWordTag(guessedTag):
			resulted_data+=self.check_word_as_verb(word_nm);
			#print "is verb", rabti,len(resulted_data);
		#if word is noun
		if self.tagger.hasNounTag(guessedTag) or self.tagger.isStopWordTag(guessedTag):			
			resulted_data+=self.check_word_as_noun(word_nm);
		if len(resulted_data)==0:
			#check the word as unkonwn
			resulted_data+=self.check_word_as_unknown(word_nm);
			#check if the word is nomralized and solution are equivalent
		resulted_data = self.check_normalized(word_vocalised, resulted_data)
		#check if the word is shadda like
		resulted_data = self.check_shadda(word_vocalised, resulted_data)

		#check if the word is vocalized like results			
		if self.partial_vocalization_support:
			resulted_data=self.check_partial_vocalized(word_vocalised, resulted_data);
		# add word frequency information in tags
		resulted_data = self.addWordFrequency(resulted_data);

		if len(resulted_data)==0:
			resulted_data.append(wordCase.wordCase({
			'word':word,  
			'affix': ('' , '', '', ''),       
			'stem':'',
			'original':word,
			'vocalized':word,
			'tags':u'',
			'type':'unknown',
			'root':'',
			'template':'',
			'freq':self.wordfreq.getFreq(word, 'unknown'),
			'syntax':'',
			})
			);
		return resulted_data;

	def setVocalized(self,newvocalized):
		"""
		Set the vocalized word
		@param newvocalized: the new given vocalized.
		@type newvocalized: unicode string
		"""
		self.vocalized  =  newvocalized;
		self.unvocalized  =  araby.stripTashkeel(newvocalized);

	def add(self, word, suggestList):
		if word!=u"" and  suggestList!=[] and  type(suggestList).__name__=='list': 
			#ToDo: adding different suggestion into one list;
			# NB: this is time eater because if the word is frequent.
			# if self.dict.has_key(word):
				# # if the dict has previous suggestions for the word,
				# # add new suggestions and remove duplicata;
				# suggestList+=self.dict[word];
				# suggestList=set(suggestList);
				# self.dict[word]=suggestList;
			#else:
			self.dict[araby.stripTashkeel(word)]=suggestList;

def vocalizeNamed(wordlist, synTags=""):
	""" Vocalize a number words
	@param wordlist: words to vocalize
	@type wordlist: unicode list
	@param synTags: tags about the clause
	@type synTags: unicode
	@return: the vocalized wordlist.
	@rtype: unicode
	"""
	newlist=[];
	prefix=u"";
	next=u"";
	#detect tags 
	# we can pass tags to this number word
	tags= synTags;
	bin_count=0;
	for i in range(len(wordlist)):
		#save the original word with possible harakat if exist
		word=wordlist[i];
		word_nm=araby.stripTashkeel(word);
		# the first word can have prefixes 
		if i==0 and word_nm:  
			# word to get majrour tag
			if word_nm in (u'أبي', u'بنو', u'آل', u'ابن',):
				tags +=u"مجرور";
			elif word_nm in (u'أبو', ):
				tags +=u"مرفوع";
			elif word_nm in (u'أبا', ):
				tags +=u"منصوب";
		# select vocalization

		if word_nm==u'بن':
			bin_count+=1;
			#treat first bin according to tags
			if bin_count==1:
				if u'مجرور' in tags:
					voc=u'بْنِ'
				elif u'مرفوع' in tags:
					voc=u'بْنُ'
				elif u'منصوب' in tags:
					voc=u'بْنَ'
				else:
					voc=u'بْن'
			else:
				#  u'مجرور' 
				voc=u'بْنِ'
		#Todo Vocalize names
		else:
			voc=word;
		newlist.append(voc);
	return newlist;

	def getUnvOriginal(self,):
		"""
		Get the unvocalized  original form of the input word
		@return: the given unvocalized original.
		@rtype: unicode string
		"""
		if self.unvoriginal:
			return self.unvoriginal;			
		else :
			if self.original:
				self.unvoriginal = araby.stripTashkeel(self.original);
			else:
				return u"";
			return self.unvoriginal;

	def getUnvocalized(self,):
		"""
		Get the unvocalized form of the input word
		@return: the given unvocalized.
		@rtype: unicode string
		"""
		if self.unvocalized:
			return self.unvocalized;
		else:
			if self.vocalized:
				self.unvocalized=araby.stripTashkeel(self.vocalized);
			else :
				return u"";
		return self.unvocalized;

	def check_normalized(self, word_vocalised, resulted_data):
		"""
		If the entred word is like the found word in dictionary, to treat some normalized cases, 
		the analyzer return the vocalized like words;
		ُIf the word is ذئب, the normalized form is ذءب, which can give from dictionary ذئبـ ذؤب.
		this function filter normalized resulted word according the given word, and give ذئب.
		@param word_vocalised: the input word.
		@type word_vocalised: unicode.
		@param resulted_data: the founded resulat from dictionary.
		@type resulted_data: list of dict.
		@return: list of dictionaries of analyzed words with tags.
		@rtype: list.
		"""
		#print word_vocalised.encode('utf8');
		filtred_data=[];
		inputword = araby.stripTashkeel(word_vocalised)
		for item in  resulted_data:
			vocalized = getattr(item, 'vocalized') 
			if vocalized:
				outputword = araby.stripTashkeel(vocalized)
				if inputword == outputword:
					filtred_data.append(item);
		return  filtred_data;

    def create_index_broken_plural(self):

        """Deprecated: create index from the broken_plural dictionary
		to accelerate the search in the dictionary for broken_plural
		"""
        for key in BrokenPluralTable.keys():
            vocnoun = key
            unvnoun = araby.stripTashkeel(vocnoun)
            normnoun = normalize_hamza(unvnoun)
            # transitive=BrokenPluralTable[key]
            # stamp=noun_stamp(normnoun);
            if self.BROKENPLURAL_DICTIONARY_INDEX.has_key(normnoun):
                self.BROKENPLURAL_DICTIONARY_INDEX[normnoun].append(vocnoun)
            else:
                self.BROKENPLURAL_DICTIONARY_INDEX[normnoun] = [vocnoun]

def getPreviousTag(word):
	"""Get the word tags
	@param word: given word
	@type word: unicode
	@return :word tag
	@rtype: unicode
	"""
	word=araby.stripTashkeel(word);
	tags=u'';
	if word in named_const.NOUN_NASEB_LIST:
		return u'منصوب';
	elif word in named_const.JAR_LIST:
		return u'مجرور';
	elif word in named_const.RAFE3_LIST:
		return u'مرفوع';
	else:
		return u'';

def treatLine(line, action):
	""" treat one line at once with action"""
	global globalFreq
	if action == "extract":
		words=araby.tokenize(line);
		for word in words:
			extract(word);
	elif action =="reduce":
		line= line.strip(' ');
		fields=line.split(' ');
		if len(fields)>=2:
			freq = fields[0]
			word = fields[1]
			word_nm = araby.stripTashkeel(word);
			if WordsTab.has_key(word_nm): # the word has multiple vocalization
				WordsTab[word_nm]=False;
			else:
				WordsTab[word_nm]={'f':freq,'v':word} ;	
			globalFreq += stringToInt(freq);

	def getSuffixVariant(self,word, suffix,enclitic):
		"""
		Get the suffix variant to be joined to the word.
		For example: word = مدرس, suffix=ة, encletic=ي. The suffix is convert to Teh.
		@param word: word found in dictionary.
		@type word: unicode.
		@param suffix: second level suffix.
		@type suffix: unicode.
		@param enclitic: first level suffix.
		@type enclitic: unicode.		
		@return: variant of suffix.
		@rtype: unicode.
		"""
		enclitic_nm=araby.stripTashkeel(enclitic)
		#if the word ends by a haraka
		if suffix.find(araby.TEH_MARBUTA)>=0 and len (enclitic_nm)>0:
			suffix=re.sub(araby.TEH_MARBUTA,araby.TEH,suffix);
		if 	enclitic_nm==u"" and word[-1:] in (araby.ALEF_MAKSURA, araby.YEH,araby.ALEF) and suffix in araby.HARAKAT  :
			suffix=u"";
		return suffix;

	def generateSuggest(self,word):
		"""
		Generate word suggestion 
		@param word: input text.
		@type word: unicode.
		@return: generated suggestion.
		rtype: list of words.
		"""
		wordlist=[word, araby.stripTashkeel(word)];
		codidates=self.edits1(word)
		for condidate in codidates:
			if True :#self.accepted(condidate):
				wordlist.append(condidate);
		# commun letters error remplacement
		for tup in spellcheck_const.TabReplacment:
			sug	=word.replace(tup[0], tup[1])
			if sug!=word: 
				# evaluate generated suggestion
				if self.accepted(sug):
					wordlist.append(sug);
		wordlist = list(set(wordlist))
		return wordlist;

	def verbStamp(self, word):
		"""
		generate a stamp for a verb, 
		the verb stamp is different of word stamp, by hamza noralization
		remove all letters which can change form in the word :
		- ALEF, 
		- YEH, 
		- WAW, 
		- ALEF_MAKSURA
		- SHADDA
		@return: stamped word
		"""
		word=araby.stripTashkeel(word);
		#The vowels are striped in stamp function
		word=araby.normalizeHamza(word);
		if word.startswith(araby.HAMZA):
			#strip The first hamza
			word=word[1:];
		# strip the last letter if is doubled
		if word[-1:]== word[-2:-1]:
			word=word[:-1];
		return self.VerbSTAMP_pat.sub('', word)