void LemmatizatorStorage_SQLITE::Lemmatize(
                                           const lem::UCString &word,
                                           lem::MCollect<lem::UCString> &lemmas
                                          )
{
 lemmas.clear();

 lem::MemFormatter mem;
 mem.printf( "SELECT L.lemma"
             " FROM lexemes_n X, lemmas L"
             " WHERE X.lexeme='%us' AND L.id=X.id_lemma", to_upper(word).c_str() );

 lem::Ptr<LS_ResultSet> rs(cnx->Select(lem::to_utf8(mem.string())));
 while( rs->Fetch() )
  {
   lemmas.push_back( rs->GetUCString(0) );
  }

 if( lemmas.empty() )
  {
   lemmas.push_back(word);
  }

 return;
}

// Генерация слов, фонетически близких к заданному word.
// Возвращается список вариантов, включая исходное слово, и список их достоверностей.
void LexicalAutomat::ProducePhonInv(
    const lem::UCString &word,
    int id_language,
    lem::MCollect<lem::UCString> &res,
    lem::MCollect<lem::Real1> &rels,
    LA_RecognitionTrace *trace
)
{
    MCollect<LA_AA_list*> packs;
    LA_AA_list *list = new LA_AA_list;
    list->reserve(16);
    list->push_back(LA_AA_item(word, Real1(100)));

    // Теперь мутированные варианты.
    LA_Pack *pack = AlephAuto(id_language, word, 1, trace);

    for (Container::size_type j = 0; j < pack->size(); j++)
    {
        const Solarix::Lexem &ph_lex = *(pack->get(j));
        if (res.find(ph_lex) == UNKNOWN)
        {
            Real1 r = pack->get(j)->get_Val();
            rels.push_back(r);
            res.push_back(ph_lex);
        }
    }

    return;
}

void SynPatternResult::FilterExportedNodes(const lem::MCollect< ExportNode > & must_be_exported)
{
    if (must_be_exported.empty())
    {
        exported_nodes.clear();
    }
    else
    {
        lem::MCollect< std::pair<const lem::UCString*, const Word_Form*> > filtered;
        for (lem::Container::size_type i = 0; i < exported_nodes.size(); ++i)
        {
            const lem::UCString & name = *exported_nodes[i].first;
            for (lem::Container::size_type j = 0; j < must_be_exported.size(); ++j)
            {
                if (must_be_exported[j].node_name == name)
                {
                    // Нашли ссылку, которую нужно перебросить в новый список, возможно уже под другим именем
                    filtered.push_back(std::make_pair(&must_be_exported[j].as_name, exported_nodes[i].second));
                    break;
                }
            }
        }

        exported_nodes = filtered;
    }

    return;
}

void Lemmatizator::Lemmatize( const lem::MCollect<lem::UCString> & words, lem::MCollect<lem::UCString> &lemmas )
{
 #if defined LEM_THREADS
 lem::Process::CritSecLocker lock(&cs);
 #endif

 if( !model_loaded )
  {
   bin->seekp( model_pos );
   model_loaded = true;
   model_available = bin->read_bool();
   if( model_available )
    {
     LoadModel();
    }
  }


 if( model_available )
  {
   LemmatizeViaModel( words, lemmas );
  }
 else
  {
   for( lem::Container::size_type i=0; i<words.size(); ++i )
    {
     lem::UCString lemma;
     Lemmatize( words[i], lemma );
     lemmas.push_back( lemma );
    }
  }

 return;
}

void LEMM_Compiler::LoadNGram( lem::Iridium::Macro_Parser & txtfile, Dictionary & dict, lem::MCollect<int> & terms, int order ) const
{
 lem::Iridium::BSourceState beg = txtfile.tellp();

 while( !txtfile.eof() )
 {
  lem::Iridium::BethToken t = txtfile.read();
  if( lem::is_int(t.string()) )
   terms.push_back( lem::to_int(t.string()) );
  else
   {
    txtfile.seekp(t);
    break;
   }
 }

 if( terms.size() != order+1 )
  {
   dict.GetIO().merr().printf( "%vfDInvalid ngram%vn\n" );
   lem::Iridium::Print_Error( beg, txtfile );
   throw lem::E_ParserError();
  }

 return;
}

void TreeMatchingExperience::AddKBCheckerMatching( int id_facts, const lem::MCollect< const Solarix::Word_Form * > & arg_values, const KB_CheckingResult & res )
{
 LEM_CHECKIT_Z( id_facts!=UNKNOWN );
 LEM_CHECKIT_Z( arg_values.size()>0 );

 TME_KBChecker * y = new TME_KBChecker( arg_values, res );
 kbid2item.insert( std::make_pair( std::make_pair(id_facts,arg_values.front()), y ) );
 return;
}

bool LA_PreprocessorRules::Crop(
    const lem::UCString &word,
    lem::MCollect<lem::UCString> &results,
    lem::MCollect<lem::Real1> &rels,
    LA_RecognitionTrace *trace
) const
{
    bool applied = false;

    if (!crop_rules.empty())
    {
        // сначала применяем префиксные правила
        typedef CROP_RULES::const_iterator IT;

        LA_CropRule::HashType prefix_hash = LA_CropRule::CalcHash(word.c_str(), true, false);
        std::pair<IT, IT> pp = prefix_crop_rules.equal_range(prefix_hash);

        lem::UCString result;

        for (auto it = pp.first; it != pp.second; ++it)
        {
            const LA_CropRule *r = it->second;
            if (r->Apply(word, result))
            {
                applied = true;
                results.push_back(result);
                rels.push_back(r->GetRel());
                if (trace != nullptr)
                {
                    trace->CropRuleApplied(word, result, r);
                }
            }
        }

        // теперь отсекаем аффикс

        LA_CropRule::HashType affix_hash = LA_CropRule::CalcHash(word.c_str(), false, true);
        pp = affix_crop_rules.equal_range(affix_hash);

        for (auto it = pp.first; it != pp.second; ++it)
        {
            const LA_CropRule *r = it->second;
            if (r->Apply(word, result))
            {
                applied = true;
                results.push_back(result);
                rels.push_back(r->GetRel());
                if (trace != nullptr)
                {
                    trace->CropRuleApplied(word, result, r);
                }
            }
        }
    }

    return applied;
}

// Ищем в справочнике набор тегов, заданный списком tags. При необходимости
// вносим в БД новую запись. Возвращается ID найденной или созданной записи.
int TagSets::Register( const lem::MCollect< std::pair<int,int> > &tags )
{
 if( tags.empty() )
  {
   return 0;
  }

 #if defined LEM_THREADS
 lem::Process::CritSecLocker lock(&cs); 
 #endif

 // Для устранения вариантов записи одного и того же набора тегов отсортируем элементы по id_tag.
 lem::MCollect< std::pair<int,int> > *sorted_tags = new lem::MCollect< std::pair<int,int> >(tags);
 std::sort( sorted_tags->begin(), sorted_tags->end(), tags_sorter );

 // Такой кортеж есть?
 const int i = tag_ptr.find(*sorted_tags);
 
 if( i==UNKNOWN )
  { 
   // Нет.
   // Поищем в БД.
   lem::UFString s;
   if( tags.size()==1 )
    {
     s = lem::format_str( L"%d %d", tags.front().first, tags.front().second );
    }
   else if( tags.size()==2 )
    {
     s = lem::format_str( L"%d %d %d %d", sorted_tags->get(0).first, sorted_tags->get(0).second, sorted_tags->get(1).first, sorted_tags->get(1).second );
    }
   else
    {
     for( lem::Container::size_type i=0; i<sorted_tags->size(); ++i )
      {
       if(i>0) s += L' ';
       s += lem::format_str( L"%d %d", sorted_tags->get(i).first, sorted_tags->get(i).second );
      }
    }
   
   const int id = db->AddTagSet(s);

   id2tags.insert( std::make_pair(id,sorted_tags) );
   tag_ptr.push_back( sorted_tags );
   tagset_id.push_back(id);

   return id;
  }
 else
  {
   delete sorted_tags;
   return tagset_id[i];
  }
}

static bool x_contains_any_of_y( const lem::MCollect<int> &x, const lem::MCollect<int> &y )
{
 if( y.size()==1 )
  return x.find(y.front())!=UNKNOWN;
 else if( y.size()==2 )
  return x.find(y.front())!=UNKNOWN || x.find(y.back())!=UNKNOWN;
 else
  {
   for( lem::Container::size_type i=0; i<y.size(); ++i )
    if( x.find(y[i])!=UNKNOWN )
     return true;

   return false;
  }
}

void SynPatternResult::SelectUnique_WithoutRemoval(lem::MCollect<const SynPatternResult*> & results)
{
    lem::MCollect<int> result_hash;
    lem::MCollect<const SynPatternResult*> unique_result;
    for (lem::Container::size_type k = 0; k < results.size(); ++k)
    {
        const SynPatternResult * result_k = results[k];
        const int h = result_k->CalcHash();

        bool found = false;
        for (lem::Container::size_type i = 0; i < unique_result.size(); ++i)
        {
            if (result_hash[i] == h)
            {
                if (SynPatternResult::Equals(result_k, unique_result[i]))
                {
                    found = true;
                    break;
                }
            }
        }

        if (!found)
        {
            result_hash.push_back(h);
            unique_result.push_back(result_k);
        }
    }

    results = unique_result;

    return;
}

void LexerTextPos::Collect_Right2Left(int count, lem::MCollect<const LexerTextPos*> & inverted_path) const
{
    inverted_path.push_back(this);
    if (count > 0 && !IsBegin() && previous != nullptr)
        GetPrev()->Collect_Right2Left(count - 1, inverted_path);

    return;
}

void SynPatternResult::GetExportCoordPairs(lem::MCollect< std::pair<int, int> > & pairs) const
{
    for (auto it = exported_coords.begin(); it != exported_coords.end(); ++it)
    {
        pairs.push_back(*it);
    }

    return;
}

void SynPatternResult::FilterExportedCoords(const lem::MCollect<int> & must_be_exported)
{
    if (must_be_exported.empty())
        exported_coords.clear();
    else
    {
        std::multimap< int /*id_coord*/, int /*id_state*/ > filtered;
        for (auto it = exported_coords.begin(); it != exported_coords.end(); ++it)
        {
            if (must_be_exported.find(it->first) != UNKNOWN)
                filtered.insert(*it);
        }

        exported_coords = filtered;
    }

    return;
}

void SynPatternResult::AppendDebugTrace(const lem::MCollect<SynPatternDebugTrace> & debug_trace2)
{
    for (lem::Container::size_type i = 0; i < debug_trace2.size(); ++i)
    {
        debug_trace.push_back(debug_trace2[i]);
    }

    return;
}

bool TreeMatchingExperience::FindKBCheckerMatching( int id_facts, const lem::MCollect< const Solarix::Word_Form * > & arg_values, KB_CheckingResult * res ) const
{
 LEM_CHECKIT_Z( id_facts!=UNKNOWN );
 LEM_CHECKIT_Z( arg_values.size()>0 );

 typedef KBID2ITEM::const_iterator IT;
 std::pair<IT,IT> pit = kbid2item.equal_range( std::make_pair( id_facts, arg_values.front() ) );

 for( IT it=pit.first; it!=pit.second; ++it )
  {
   if( arg_values == it->second->arg_values )
    {
     *res = it->second->res;
     return true;
    }
  }

 return false;
}

Word_Form::Word_Form( const lem::MCollect<const Word_Form*> &variants )
{
 LEM_CHECKIT_Z( !variants.empty() );

 // Первая версия становится основной, ее не копируем в альтернативы.
 for( lem::Container::size_type i=1; i<variants.size(); ++i )
  alt.push_back( new Word_Form(*variants[i]) );

 name = variants[0]->name;
 normalized = variants[0]->normalized;
 pair = variants[0]->pair;
 entry_key = variants[0]->entry_key;
 val = variants[0]->val;
 score=variants[0]->score;
 origin_pos = variants[0]->origin_pos;
 tokenizer_flags = variants[0]->tokenizer_flags;

 iversion = seq_iversion++;
 return;
}

void LexerTextPos::CollectPathToLeft(int count, lem::MCollect<const Word_Form*> & org) const
{
    LEM_CHECKIT_Z(count >= 0);

    org.push_back(wordform);

    if (count > 0 && previous != NULL)
        previous->CollectPathToLeft(count - 1, org);

    return;
}

void LexerTextPos::Collect_Right2Left(const LexerTextPos *left_boundary, lem::MCollect<const LexerTextPos*> & inverted_path) const
{
    LEM_CHECKIT_Z(left_boundary != nullptr);

    inverted_path.push_back(this);

    if (this != left_boundary && previous != nullptr)
        previous->Collect_Right2Left(left_boundary, inverted_path);

    return;
}

void SG_DeclensionTable::GenerateForms(
                                       const Lexem &entry_name,
                                       lem::MCollect<Lexem> &res,
                                       const SynGram &sg,
                                       const SG_DeclensionAutomat &dsa 
                                      ) const
{
 res.reserve(form.size());

 for( lem::Container::size_type i=0; i<form.size(); i++ )
  {
   UCString frm( dsa.ProduceForm( entry_name, GetClass(), *form[i], sg ) );

   // Без повторов
   if( std::find( res.begin(), res.end(), frm )==res.end() )
    res.push_back( frm );
  }

 return;
}

void SynPatterns::GetUnresolvedForwardDeclarations( lem::MCollect<lem::UCString> & unresolved_names ) const
{
 typedef std::map< lem::UCString, int >::const_iterator IT;
 for( IT it=name2id.begin(); it!=name2id.end(); ++it )
  {
   if( id2count.find( it->second )==id2count.end() )
    unresolved_names.push_back( it->first );
  }

 return;
}