def _get_ttype_class(ttype):
	fname = STANDARD_TYPES.get(ttype)
	if fname:
		return fname
	aname = ''
	while fname is None:
		aname = '-' + ttype[-1] + aname
		ttype = ttype.parent
		fname = STANDARD_TYPES.get(ttype)
	return fname + aname

def _get_ttype_name(ttype):
    fname = STANDARD_TYPES.get(ttype)
    if fname:
        return fname
    aname = ""
    while fname is None:
        aname = ttype[-1] + aname
        ttype = ttype.parent
        fname = STANDARD_TYPES.get(ttype)
    return fname + aname

    def _output_token(self, ttype, value, pos, outfile):
        # Manually split things like the "os.path" in "import os.path" into separate tokens so we can annotate them separately
        if ttype == Token.Name.Namespace and '.' in value:
            names = value.split('.')
            r, c = pos
            for i, n in enumerate(names):
                if i:
                    self._output_token(Token.Text, u'.', (r, c), outfile)
                    c += 1
                self._output_token(Token.Name.Namespace, n, (r, c), outfile)
                c += len(n)
            return

        if ttype == Token.Text and pos[1] > 0:
            if '\n' in value:
                outfile.write('</span>')
                self.current_errors = []

        id_str = ' id="%d_%d"' % pos

        cls_str = ''
        cls = STANDARD_TYPES.get(ttype)
        classes = []
        if cls:
            classes.append(cls)
        type_idx = ''
        if ttype in Token.Name:
            classes.append("anno")
            classes.append("usedef-" + value.encode("base64").replace('=', '').replace('\n', ''))

            # print pos, ttype
            node = self.pos_nodes.get(pos, None)
            u = self.node_types.get(node, None)
            if u:
                type_idx = ' type_idx="%s"' % ','.join(str(self._get_type_idx(t)) for t in u.types())
            else:
                print "missed", pos, node
        if classes:
            cls_str = ' class="%s"' % ' '.join(classes)

        outfile.write('<span%s%s%s>' % (cls_str, id_str, type_idx))

        translated_val = value.translate(self._html_escape_table)
        outfile.write(translated_val.encode("utf8"))

        outfile.write('</span>')

 def format(self, tokenstream, outfile):
     row = 1
     column = 1
     for (token_type, text) in tokenstream:
         parts = text.split("\n")
         new_row = row + len(parts) - 1
         if len(parts) > 1:
             new_column = len(parts[-1])
         else:
             new_column = column + len(parts[-1]) - 1
         if token_type is not Token.Text:
             self._annotation_list.append(Annotation(
                 row,
                 column,
                 new_row,
                 new_column,
                 {
                     "type": "style",
                     "what": STANDARD_TYPES.get(token_type, ""),
                 }
             ))
         row = new_row
         column = new_column + 1

def get_pygments_tokens(page, elem, uid):
    """inserts a table containing all existent token types and corresponding
       css class, with an example"""
    # The original div in the raw html page may contain some text
    # as a visual reminder that we need to remove here.
    elem.text = ''
    elem.attrib['class'] = CRUNCHY_PYGMENTS
    table = SubElement(elem, 'table')
    row = SubElement(table, 'tr')
    for title in ['Token type', 'css class']:
        column = SubElement(row, 'th')
        column.text = title
    keys = list(STANDARD_TYPES.keys())
    keys.sort()
    for token in keys:
        if len(repr(token)) == 5: # token = Token
            continue
        row = SubElement(table, 'tr')
        column1 = SubElement(row, 'td')
        column1.text = repr(token)[6:] # remove "Token."
        column2 = SubElement(row, 'td')
        token_class = STANDARD_TYPES[token]
        column2.text = token_class.split('_')[0]
        column3 = SubElement(row, 'td')
        span = SubElement(column3, 'span')
        span.attrib['class'] = token_class
        span.text = " * test * "
        column4 = SubElement(row, 'td')
        _code = SubElement(column4, 'code')
        _code.attrib['class'] = token_class
        _code.text = " * test * "
        column5 = SubElement(row, 'td')
        var = SubElement(column5, 'var')
        var.attrib['class'] = token_class
        var.text = " * test * "
    return

def line_out(out, line, side=None):
	if len(line) >= 4:
		dtype, num, chunks, syntax_chunks = line
	else:
		dtype, num, chunks = line
		syntax_chunks = None
	if not chunks:
		ltype = "nil"
	elif dtype == -1:
		ltype = "old"
	elif dtype == 1:
		ltype = "new"
	else:
		ltype = "same"

	if side:
		side = "%s " % side
	else:
		side = ""

	errors = []

	out.extend('<td class="%s%s num">' % (side, ltype))
	if num is not None:
		out.extend(smart_str(escape(num)))
	out.extend('</td>')
	out.extend('<td class="%s%s line">' % (side, ltype))
	if chunks:
		for ddt, text in chunks:
			if ddt == -1:
				out.extend('<del>')
			elif ddt == 1:
				out.extend('<ins>')
			if syntax_chunks:
				syntax_chunks = list(syntax_chunks)
				while len(text) > 0:
					if len(syntax_chunks) <= 0:
						error = "Chunk overflow error '%s' does not have a syntax chunk" % smart_str(text)
						print error
						errors.append(error)
						out.extend(line_to_html(text))
						text = ''
						continue
					syntax_type, syntax_text = syntax_chunks.pop(0)
					if len(syntax_text) > len(text):
						# If the syntax chunk is larger than the diff chunk then prepend a new syntax chunk with the remainder
						syntax_chunks.insert(0, (syntax_type, syntax_text[len(text):]))
						syntax_text = syntax_text[:len(text)]
					subchunk_text = text[:len(syntax_text)]
					text = text[len(syntax_text):] # Trim the initial text off the chunk text
					if syntax_text == subchunk_text:
						cls = ''
						fname = STANDARD_TYPES.get(syntax_type)
						if fname:
							cls = fname
						else:
							aname = ''
							while fname is None:
								aname = '-' + syntax_type[-1] + aname
								syntax_type = syntax_type.parent
								fname = STANDARD_TYPES.get(syntax_type)
							cls = fname + aname

						if cls:
							out.extend('<span class="%s">' % " ".join(cls))
							out.extend(line_to_html(subchunk_text))
							out.extend('</span>')
						else:
							out.extend(line_to_html(subchunk_text))
					else:
						# Chunk mismatch (code error)
						error = "Chunk mismatch error '%s' does not match '%s'" % (smart_str(syntax_text), smart_str(subchunk_text))
						print error
						errors.append(error)
						out.extend(line_to_html(subchunk_text))

			else:
				out.extend(line_to_html(text))
			if ddt == -1:
				out.extend('</del>')
			elif ddt == 1:
				out.extend('</ins>')
	for error in errors:
		out.extend('<br>')
		out.extend('<span class="error differror">')
		out.extend(smart_str(escape(error)))
		out.extend('</span>')
	out.extend('</td>')

 def _append(self, type, value, element):
     class_ = STANDARD_TYPES.get(type)
     if class_:
         value = moin_page.span(attrib={moin_page.class_: class_}, children=(value, ))
     element.append(value)

    def _output_token(self, ttype, value, pos, outfile):
        # Manually split things like the "os.path" in "import os.path" into separate tokens so we can annotate them separately
        if ttype == Token.Name.Namespace and '.' in value:
            names = value.split('.')
            r, c = pos
            for i, n in enumerate(names):
                if i:
                    self._output_token(Token.Text, u'.', (r, c), outfile)
                    c += 1
                self._output_token(Token.Name.Namespace, n, (r, c), outfile)
                c += len(n)
            return

        outfile.write('<span class="error">' * self._num_errors_start(pos))

        outfile.write('</span>' * self._num_errors_end(pos))

        if ttype == Token.Text and pos[1] > 0:
            if '\n' in value:
                outfile.write('</span>')
                self.current_errors = []

        id_str = ''
        if self._have_type_info_for_pos(pos):
            id_str = ' id="%d_%d"' % pos

        def output_preview(should_slide=False):
            slide_class = ' slide' if should_slide else ''
            outfile.write('<span class="anno_preview%s" id="col_%d"></span>' % (slide_class, pos[0]))
            self._output_errors_for_line(pos[0], outfile)

        # This currently outputs errors and annotations before comments
        if ttype == Token.Comment and pos[1] > 0:
            self.comments.add(pos[0])
            output_preview(True)
        elif ttype == Token.Text and pos[1] > 0 and pos[0] not in self.comments:
            try:
                value.index('\n')
                should_slide = len(self._errors_for_line(pos[0])) > 0
                output_preview(should_slide)
            except ValueError:
                pass
            
        cls_str = ''
        cls = STANDARD_TYPES.get(ttype)
        if cls or id_str:
            cls_str = ' class="'
            if id_str:
                cls_str += 'anno '
            if cls:
                cls_str += cls
            cls_str += '"'

        if pos in self.links:
            outfile.write("<a href='%s'>" % self.links[pos])

        if cls_str or id_str:
            outfile.write('<span%s%s>' % (cls_str, id_str))

        translated_val = value.translate(self._html_escape_table)
        outfile.write(translated_val.encode("utf8"))

        if cls:
            outfile.write('</span>')

        if pos in self.links:
            outfile.write("</a>")

'''

CLASS_START = '''class %sStyle(Style):

    """Pygments style %sStyle."""

    background_color = "%s"  %s
    highlight_color = "%s"   %s

    styles = {
%s
    }
'''

tokens = {"hll": "highlight_color"}
for k, v in ST.items():
    if v == "":
        continue
    tokens[v] = str(k).replace("Token.", '')


class Comments(object):

    """Comment strip class."""

    re_line_preserve = re.compile(r"\r?\n", re.MULTILINE)
    re_css_comment = re.compile(
        r'''(?x)
            (?P<comments>
                /\*[^*]*\*+(?:[^/*][^*]*\*+)*/  # multi-line comments
              | \s*//(?:[^\r\n])*               # single line comments