def add_VerbForm(word):
    fmap = word.feat_map()

    if word.cpostag not in VERB_TAGS:
        return []

    # (see https://github.com/TurkuNLP/UniversalFinnish/issues/28)
    if 'INF' in fmap:
        # infinitive
        assert 'PCP' not in fmap, 'INF and PCP'
        assert 'PRS' not in fmap, 'INF and PRS'
        assert 'MOOD' not in fmap, 'INF and MOOD'
        value = fmap['INF']
        if value in ('Inf1', 'Inf2', 'Inf3'):
            return [('VerbForm', 'Inf')]
        else:
            warn(u'unexpected INF value ' + value)
            return []
    if 'PCP' in fmap:
        # participle
        assert 'INF' not in fmap, 'PCP and INF'
        assert 'PRS' not in fmap, 'PCP and PRS'
        assert 'MOOD' not in fmap, 'PCP and MOOD'
        return [('VerbForm', 'Part')]
    else:
        # Should be finite, check for some marker. We consider any
        # non-infinitive, non-participle verb finite if it has either
        # MOOD or PRS.
        # (https://github.com/TurkuNLP/UniversalFinnish/issues/28)
        if 'MOOD' in fmap or 'PRS' in fmap:
            return [('VerbForm', 'Fin')]
        else:
            warn(u'failed to assign VerbForm to ' + unicode(word))
            return []

 def test_json_HTTPResponse(self):
     self.app.route('/')(lambda: py3web.HTTPResponse({'a': 1}, 500))
     try:
         self.assertBody(py3web.json_dumps({'a': 1}))
         self.assertHeader('Content-Type','application/json')
     except ImportError:
         warn("Skipping JSON tests.")

def rewrite_SUBCAT(word):
    for tagset, func in subcat_rewrite_func:
        if word.cpostag in tagset:
            return func(word)

    warn(word.cpostag + " with SUBCAT")
    return []

 def setUp(self):
     if self.skip: return
     # Find a free port
     for port in range(8800, 8900):
         self.port = port
         # Start servertest.py in a subprocess
         cmd = [sys.executable, serverscript, self.server, str(port)]
         cmd += sys.argv[1:] # pass cmdline arguments to subprocesses
         self.p = Popen(cmd, stdout=PIPE, stderr=PIPE)
         # Wait for the socket to accept connections
         for i in xrange(100):
             time.sleep(0.1)
             # Accepts connections?
             if ping('127.0.0.1', port): return
             # Server died for some reason...
             if not self.p.poll() is None: break
         rv = self.p.poll()
         if rv is None:
             raise AssertionError("Server took to long to start up.")
         if rv is 128: # Import error
             tools.warn("Skipping %r test (ImportError)." % self.server)
             self.skip = True
             return
         if rv is 3: # Port in use
             continue
     raise AssertionError("Could not find a free port to test server.")

def rewrite_PRON_SUBCAT(word):
    value = word.feat_map()["SUBCAT"]

    if value == "Dem":  # demonstrative
        return [("PronType", "Dem")]
    elif value == "Pers":  # personal
        return [("PronType", "Prs")]
    elif value == "Rel":  # relative
        return [("PronType", "Rel")]
    elif value == "Indef":  # indefinite
        return [("PronType", "Ind")]
    elif value == "Interr":  # interrogative
        return [("PronType", "Int")]
    elif value == "Recipr":  # reciprocal
        return [("PronType", "Rcp")]
    elif value == "Refl":  # reflexive
        # NOTE: UD defines Reflexive as a separate feature from PronType
        # (http://universaldependencies.github.io/docs/u/feat/Reflex.html)
        # TODO: consider adding PronType also?
        return [("Reflex", "Yes")]
    elif value == "Qnt":
        # NOTE: UD does not define "quantifier" as a pronoun type, so
        # these are (tentatively) mapped to the closest corresponding
        # subcategory, indefinite pronouns.
        # see https://github.com/TurkuNLP/UniversalFinnish/issues/37
        warn("mapping PRON SUBCAT " + value + " to Ind")
        return [("PronType", "Ind")]
    else:
        return []

def rewrite_MOOD(word):
    value = word.feat_map()["MOOD"]

    if value == "Ind":  # indicative
        return [("Mood", "Ind")]
    elif value == "Imprt":  # imperative
        return [("Mood", "Imp")]
    elif value == "Cond":  # conditional
        return [("Mood", "Cnd")]
    elif value == "Pot":  # potential
        return [("Mood", "Pot")]
    elif value == "Opt":  # optative
        # Omorfi defines the archaic optative mood (e.g. "kävellös"),
        # which is extremely rare (e.g. no occurrences in TDT). Should
        # this ever appear, we will approximate as imperative.
        warn("mapping optative mood to imperative")
        return [("Mood", "Imp")]
    elif value == "Eve":  # eventive
        # Omorfi defines the archaic eventive mood (e.g. "kävelleisin"),
        # which is extremely rare (e.g. no occurrences in TDT). Should
        # this ever appear, we will approximate as potential.
        warn("mapping eventive mood to potential")
        return [("Mood", "Pot")]
    else:
        return []  # assert False, 'unknown MOOD value ' + value

 def test_json(self):
     self.app.route('/')(lambda: {'a': 1})
     try:
         self.assertBody(bottle.json_dumps({'a': 1}))
         self.assertHeader('Content-Type','application/json')
     except ImportError:
         warn("Skipping JSON tests.")

def rewrite_A(word):
    # Assign "Pron" to instances of particular words given "A"
    # analyses by lemma.
    # (see https://github.com/TurkuNLP/UniversalFinnish/issues/66)
    if word.lemma in ('muu', 'sama'):
        warn('assigning %s to Pron/PRON' % word.form)
        return ('PRON', 'Pron')

    return 'ADJ'

def rewrite_VOICE(word):
    if word.cpostag not in VERB_TAGS:
        warn(word.cpostag + " with VOICE")

    value = word.feat_map()["VOICE"]
    if value == "Act":
        return [("Voice", "Act")]
    elif value == "Pass":
        return [("Voice", "Pass")]
    else:
        return []  # assert False, 'unknown VOICE value %s' % value

def rewrite_TENSE(word):
    if word.cpostag not in VERB_TAGS:
        warn(word.cpostag + " with TENSE")

    value = word.feat_map()["TENSE"]
    if value == "Prs":
        return [("Tense", "Pres")]
    elif value == "Prt":
        return [("Tense", "Past")]
    else:
        return []  # assert False, 'unknown TENSE value %s' % value

 def test_json_serialization_error(self):
     """
     Verify that 500 errors serializing dictionaries don't return
     content-type application/json
     """
     self.app.route('/')(lambda: {'a': set()})
     try:
         self.assertStatus(500)
         self.assertHeader('Content-Type','text/html; charset=UTF-8')
     except ImportError:
         warn("Skipping JSON tests.")

def rewrite_Punct(word):
    # NOTE: likely not a final mapping, see
    # https://github.com/TurkuNLP/UniversalFinnish/issues/1
    if is_symbol(word.form):
        assert not is_punctuation(word.form), 'internal error'
        return 'SYM'
    elif is_punctuation(word.form):
        assert not is_symbol(word.form), 'internal error'
        return 'PUNCT'
    else:
        warn(u'assigning SYM to unrecognized word ' + word.form)
        return 'SYM'

def rewrite_Pron(word):
    # Assign "A" to pro-adjectives such as "millainen" based on lemma
    # (see https://github.com/TurkuNLP/UniversalFinnish/issues/67).
    if word.lemma in pro_adjective_lemmas:
        warn('assigning %s to A/ADJ' % word.form)
        return ('ADJ', 'A')

    # NOTE: this is not a full mapping: some words tagged Pron should
    # map into DET instead. See
    # https://github.com/TurkuNLP/UniversalFinnish/issues/1,
    # https://github.com/TurkuNLP/UniversalFinnish/issues/27,
    # https://github.com/UniversalDependencies/docs/issues/97.
    # However, we're currently postponing this exception.
    return 'PRON'

def rewrite_CMP(word):
    value = word.feat_map()["CMP"]

    if word.cpostag not in (ADJ_TAGS | VERB_TAGS | ADV_TAGS):
        warn(word.cpostag + " with CMP")

    if value == "Comp":
        return [("Degree", "Cmp")]
    elif value == "Pos":
        return [("Degree", "Pos")]
    elif value == "Superl":
        return [("Degree", "Sup")]
    else:
        return []  # assert False, 'unknown CMP value ' + value

def rewrite_pos(sentence):
    for w in sentence.words():
        try:
            rewritten = rewrite_func[w.cpostag](w)
        except KeyError:
            warn(u'unexpected cpostag ' + w.cpostag)
            assert False, 'unexpected cpostag ' + w.cpostag

        # if rewrite_func returns a tuple, assign both cpostag and
        # postag; otherwise assign only cpostag
        if isinstance(rewritten, tuple):
            w.cpostag, w.postag = rewritten
        else:
            w.cpostag = rewritten

def add_SUBCAT_to_Pron(word):
    # Assign feature SUBCAT to pronouns lacking it using surface
    # form-based heuristics.

    fmap = word.feat_map()

    if 'SUBCAT' in fmap or word.postag != 'Pron':
        return []

    try:
        sc = Pron_SUBCAT_by_lemma[word.lemma]
    except KeyError:
        warn(u'failed to assign SUBCAT to Pron')
        return []

    return [('SUBCAT', sc)]

 def tearDown(self):
     if self.skip: return
     for i in range(10):
         if self.p.poll() != None: break
         os.kill(self.p.pid, signal.SIGINT)
         time.sleep(0.1*i)
     for i in range(10):
         if self.p.poll() != None: break
         os.kill(self.p.pid, signal.SIGINT)
         time.sleep(i)
     for stream in (self.p.stdout, self.p.stderr):
         for line in stream:
             if tob('warning') in line.lower():
                 tools.warn(line.strip().decode('utf8'))
             elif tob('error') in line.lower():
                 raise AssertionError(line.strip().decode('utf8'))

    def tearDown(self):
        if self.skip: return

        if self.p.poll() == None:
            os.kill(self.p.pid, signal.SIGINT)
            time.sleep(0.5)
        while self.p.poll() == None:
            os.kill(self.p.pid, signal.SIGTERM)
            time.sleep(1)

        for stream in (self.p.stdout, self.p.stderr):
            for line in stream:
                if tob('warning') in line.lower():
                    tools.warn(line.strip().decode('utf8'))
                elif tob('error') in line.lower():
                    raise AssertionError(line.strip().decode('utf8'))

def rewrite_NUM(word):
    fmap = word.feat_map()

    if word.cpostag not in (VERB_TAGS | NOUN_TAGS | ADJ_TAGS | PRON_TAGS):
        warn(word.cpostag + " with NUM")

    # Both PRS and NUM would generate redundant Number features
    assert "PRS" not in fmap

    value = word.feat_map()["NUM"]
    if value == "Sg":
        return [("Number", "Sing")]
    elif value == "Pl":
        return [("Number", "Plur")]
    else:
        # assert False, 'unknown NUM value %s' % value
        return []

def add_Person(word):
    # Assign feature Person to personal pronouns, which for some
    # reason lack it in Omorfi analyses.

    fmap = word.feat_map()

    if word.cpostag != 'PRON':
        return []
    elif fmap.get('SUBCAT') != 'Pers':
        return []
    else:
        p = person_by_lemma.get(word.lemma, None)
        if p is not None:
            return [('Person', p)]
        else:
            warn(u'missing person for pronoun lemma ' + word.lemma)
            return []