class ItemTypeExtensionTest(unittest.TestCase):

  def setUp(self):
    self.md = Markdown([ItemTypeExtension()])

  def test_beginning(self):
    source = "* [deferred] элемент списка";
    expectedRe = r"""^<ul>\s*<li class="type_deferred">элемент списка</li>\s*</ul>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_middle(self):
    source = "* элемент [wait] списка";
    expectedRe = r"""^<ul>\s*<li class="type_wait">элемент +списка</li>\s*</ul>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_end(self):
    source = "* элемент списка [freetime]";
    expectedRe = r"""^<ul>\s*<li class="type_freetime">элемент списка *</li>\s*</ul>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_unknown_is_not_replaced(self):
    source = "* элемент [списка]";
    expectedRe = r"""^<ul>\s*<li>элемент \[списка\]</li>\s*</ul>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_references_not_messed_up(self):
    source = "Тестовая [freetime][deferred] ссылка\n\n[deferred]: http://example.com"
    expectedRe = r"""^<p>Тестовая <a href=["']http://example.com["']>freetime</a> ссылка</p>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

class MarkdownReader(Reader):
    enabled = bool(Markdown)
    file_extensions = ['md', 'markdown', 'mkd', 'mdown']
    default_extensions = ['codehilite(css_class=highlight)', 'extra']

    def __init__(self, *args, **kwargs):
        super(MarkdownReader, self).__init__(*args, **kwargs)
        self.extensions = self.settings.get('MD_EXTENSIONS',
                                            self.default_extensions)
        self.extensions.append('meta')
        self._md = Markdown(extensions=self.extensions)

    def _parse_metadata(self, meta):
        """Return the dict containing document metadata"""
        output = {}
        for name, value in meta.items():
            name = name.lower()
            if name == "summary":
                summary_values = "\n".join(value)
                # reset the markdown instance to clear any state
                self._md.reset()
                summary = self._md.convert(summary_values)
                output[name] = self.process_metadata(name, summary)
            else:
                output[name] = self.process_metadata(name, value[0])
        return output

    def read(self, source_path):
        """Parse content and metadata of markdown files"""

        with pelican_open(source_path) as text:
            content = self._md.convert(text)

        metadata = self._parse_metadata(self._md.Meta)
        return content, metadata

def load_stories(story_stuff):
    meta = {
        "title": (True, ("one",)),
        "description": (True, ("one",)),
        "image": (True, ("one",)),
        "button": (True, ("one",)),
        "link": (True, ("one",)),
        "button2": (False, ("one",)),
        "link2": (False, ("one",)),
        "action": (False, ("one",)),
    }
    markdowner = Markdown(extensions=["meta"], output_format="html5")
    stories = []
    for story_file, filename in story_stuff:
        story = {}
        html = markdowner.convert(story_file.read())  # also loads metadata
        story["body"] = Markup(html)
        story["slug"] = os.path.splitext(filename)[0]

        for field, (required, filters) in meta.items():
            field_val = markdowner.Meta.get(field)
            try:
                val = apply_field_constraints(field_val, required, filters)
            except MetaError as e:
                e.apply_context(filename, field)
                raise e
            story[field] = val

        stories.append(story)
        markdowner.reset()

    return stories

    def read(self, filename):
        """Parses the given file and returns a :class:`Page` object

        :param filename: path to the file to read.
        :type filename: str.
        :returns: Page
        """
        try:
            text = open(filename, 'r', encoding='utf-8').read()
        except UnicodeDecodeError:
            print('wrong encoding: {0}'.format(filename))
            raise
        md = Markdown(extensions=set(self.extensions + ['meta']))
        content = md.convert(text)

        metadata = {}
        for name, value in md.Meta.items():
            name = name.lower()
            metadata[name] = self.process_metadata(name, value[0])
        return Page(filename.replace(self.path, ''),
                    content,
                    metadata.get('title'),
                    metadata.get('title_short'),
                    metadata.get('template'),
                    metadata.get('parent'),
                    metadata.get('sort'),
                    metadata.get('in_nav'))

class TableMarkupExtensionTest(unittest.TestCase):

  def setUp(self):
    self.originalMd = Markdown()
    self.md = Markdown([TableMarkupExtension()])

  def test_basic(self):

    firstPart = "First part with two paragraphs\n\nThe second paragraph\n\n* A list"
    secondPart = "Second part"

    actual = self.md.convert(
      firstPart + \
      "\n\n-----\n\n" + \
      secondPart)
    expectedRe = \
      r"^<tr>\s*" + \
        r"<td>\s*" + re.escape(self.originalMd.convert(firstPart)) + "\s*</td>\s*" + \
        r"<td>\s*" + re.escape(self.originalMd.convert(secondPart)) + "\s*</td>\s*" + \
      r"</tr>$"""
    self.assertRegex(actual, expectedRe)

  def test_beginning_hr_is_ignored(self):
    source = "-----\n\nТекст"
    expectedRe = r"""^<tr>\s*<td>\s*<p>Текст</p>\s*</td>\s*</tr>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_ending_hr_is_ignored(self):
    source = "Текст\n\n-----"
    expectedRe = r"""^<tr>\s*<td>\s*<p>Текст</p>\s*</td>\s*</tr>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

    def convert(self, source, **context):
        """ Converts the source file and saves to the destination """
        with codecs.open(source, encoding='utf-8') as src:
            lines = src.readlines()

        # Parse metadata first so we can get theme extensions
        md = Markdown()
        lines = MetaPreprocessor(md).run(lines)

        Meta = md.Meta
        meta = {k: ' '.join(v) for k, v in Meta.items()}

        # Load theme from meta data if set
        theme = meta.get('theme', 'default')

        exts = self.config.theme_get(theme, 'markdown_extensions', [
            'codehilite(css_class=syntax,guess_lang=False)'])
        exts = filter(None, exts)  # Removes empty lines

        md = Markdown(extensions=exts)
        md.Meta = meta  # restore already parsed meta data

        content = md.convert(''.join(lines))

        context['Meta'] = Meta
        context['meta'] = meta

        return self.config.render_template(theme, content, **context)

def mk_manual():
    import codecs
    md = Markdown(extensions=['toc'])
    content = md.convert(codecs.open('readme.md', encoding='utf8').read())
    with open('summary/asema-manual.html','w') as out:
        out.write(html_manual.substitute(content=content.encode('utf8')))
    upload('summary/asema-manual.html')

    def load_entries(self):
        md = Markdown(extensions=['codehilite(guess_lang=False)', 'meta', LightBoxExtension()])
        raw = (file(fn, 'r').read().decode('utf-8') for fn in glob('entries/*'))

        self.entries = []
        self.galley_entries = []

        for entry in raw:
            html, meta = md.convert(entry), md.Meta
            if 'title' not in meta or 'date' not in meta:
                continue
            title, date = meta['title'][0], meta['date'][0]
            tags = meta.get('tags', [])
            print 'Processed', title

            this_entry = dict(
                title=title,
                date=date,
                tags=tags,
                raw=entry,
                html=html,
                link=date + '_' + self.title_sub(title) + '.html'
            )

            if 'galley' in tags:
                self.galley_entries.append(this_entry)
            else:
                self.entries.append(this_entry)

        self.entries.sort(lambda a, b: cmp(b['date'], a['date']))
        self.galley_entries.sort(lambda a, b: cmp(b['date'], a['date']))

    def test_i18n_always_after_toc(self):
        text = '# This is h1'
        expected_toc = (
            '<div class="toc">'
            '    <ul>'
            '        <li><a href="#esto-es-h1">Esto es h1</a></li>'
            '    </ul>'
            '</div>'
        )

        self.catalog.add(
            'This is h1',
            'Esto es h1'
        )
        self.write_po()

        md = Markdown(
            extensions=['markdown.extensions.toc', 'markdown_i18n'],
            extension_configs={
                'markdown_i18n': {
                    'i18n_dir': self.dir,
                    'i18n_lang': 'es_ES'
                }
            }
        )

        md.convert(text)
        toc = getattr(md, 'toc', '')
        self.assertEqual(clean_xml(toc), clean_xml(expected_toc))

    def read(self, filename):
        """Parse content and metadata of markdown files"""
        # parse Rmd file - generate md file
        md_filename = filename.replace(".Rmd", ".aux").replace(".rmd", ".aux")
        robjects.r(
            """
require(knitr);
opts_knit$set(base.dir='{2}/content');
knit('{0}', '{1}', quiet=TRUE, encoding='UTF-8');
""".format(
                filename, md_filename, settings.DEFAULT_CONFIG.get("PATH")
            )
        )
        # parse md file
        md = Markdown(extensions=["meta", "codehilite(css_class=highlight)", "extra"])
        with pelican_open(md_filename) as text:
            content = md.convert(text)
        os.remove(md_filename)
        # find metadata
        metadata = {}
        for name, value in md.Meta.items():
            name = name.lower()
            meta = self.process_metadata(name, value[0])
            metadata[name] = meta
        return content, metadata

def main(in_file_path, out_file_path):
    with open(in_file_path, 'r') as in_file, \
            open(out_file_path, 'w') as out_file:
        converter = Markdown(output_format='html5')
        markdown = in_file.read()
        html = converter.convert(markdown)
        out_file.write(html)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-o", "--output", help="Output pickle file path",
                        default=None)
    parser.add_argument("input", help="Input API Blueprint file")
    args = parser.parse_args()
    with codecs.open(args.input, "r", "utf-8") as fin:
        txt = fin.read()
    m = Markdown(extensions=["plueprint"])
    m.set_output_format("apiblueprint")
    api = m.convert(txt)
    if args.output is not None:
        with open(args.output, "wb") as fout:
            pickle.dump(api, fout, protocol=-1)
    else:
        print(api)
        print("Resource groups:")
        for g in api:
            print("  %s" % g)
            print("  Resources:")
            for r in g:
                print("    %s" % r)
                print("    Actions:")
                for a in r:
                    print("      %s" % a)

    def test_callbacks(self):
        def dont_linkify_python(attrs, new=False):
            if not new:  # This is an existing <a> tag, leave it be.
                return attrs

            # If the TLD is '.py', make sure it starts with http: or https:
            text = attrs['_text']
            if text.endswith('.py') and \
               not text.startswith(('www.', 'http:', 'https:')):
                # This looks like a Python file, not a URL. Don't make a link.
                return None

            # Everything checks out, keep going to the next callback.
            return attrs

        configs = {
            'linkifycallbacks': [[dont_linkify_python], '']
        }
        linkify_ext = LinkifyExtension(configs=configs)
        md = Markdown(extensions=[linkify_ext])

        text = "setup.com www.setup.py http://setup.py setup.py"
        expected = ('<p><a href="http://setup.com">setup.com</a> '
                    '<a href="http://www.setup.py">www.setup.py</a> '
                    '<a href="http://setup.py">http://setup.py</a> '
                    'setup.py</p>')
        actual = md.convert(text)
        self.assertEqual(expected, actual)

class UrlizeExtensionTest(unittest.TestCase):

  def setUp(self):
    self.md = Markdown([UrlizeExtension()])

  def test_links_in_angle_brackets_processed(self):
    source = "<http://yandex.ru/>"
    expectedRe = r"""^<p><a href=['"]http://yandex.ru/['"]>http://yandex.ru/</a></p>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_links_starting_with_http_processed(self):
    source = "http://example.com/"
    expectedRe = r"""^<p><a href=['"]http://example.com/['"]>http://example.com/</a></p>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_links_starting_with_www_processed(self):
    source = "www.example.com"
    expectedRe = r"""^<p><a href=['"]http://www.example.com['"]>www.example.com</a></p>$"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_markdown_links_not_messed_up(self):
    source = "[](http://yandex.ru/)"
    expectedRe = r""" href=['"]http://yandex.ru/['"]"""
    self.assertRegex(self.md.convert(source), expectedRe)

  def test_manual_links_not_messed_up(self):
    source = "<a href='http://yandex.ru/'>Яндекс</a>"
    expected = "<p>%s</p>" % source
    self.assertEqual(expected, self.md.convert(source))

 def __init__(self, templatefile, *apifiles, **kwargs):
     if len(apifiles) == 0:
         raise ValueError("There must be at least one APIBlueprint file "
                          "specified")
     include_comments = kwargs.pop("include_comments", True)
     base_class = kwargs.pop("base_class", None)
     if base_class is None:
         base_class = "rest_framework.test.APITestCase"
     self._use_html2text = kwargs.pop("html2text", False)
     m = Markdown(extensions=["plueprint"])
     m.set_output_format("apiblueprint")
     with codecs.open(apifiles[0], "r", "utf-8") as fin:
         self._api = m.convert(fin.read())
     for f in apifiles[1:]:
         with codecs.open(f, "r", "utf-8") as fin:
             self._api.merge(m.convert(fin.read()))
     env = Environment(loader=FileSystemLoader(templatefile),
                       autoescape=False, trim_blocks=True,
                       lstrip_blocks=True,
                       extensions=("jinja2.ext.loopcontrols",))
     env.filters["symbolize"] = self._symbolize
     env.filters["html2text"] = self._html2text
     self._template = env.get_template("")
     self._include_comments = include_comments
     self._base_class = base_class[base_class.rfind('.') + 1:]
     self._base_module = base_class[:-len(self._base_class) - 1]
     self._counter = 1

def get_bug(filename):
    print 'Getting for {}'.format(filename)
    if filename[-4:] != '.txt':
         filename = filename + '.txt'
    with codecs.open(os.path.join(bugsdir, filename), 'r', 'utf-8-sig') as f:
        text = f.read()
    md = Markdown(extensions=['markdown.extensions.meta'],
                  output_format='html5')
    output={'content':md.convert(text)}
    output['title'] = md.Meta.get('title', [''])[0]
    output['id'] = filename[:-4]
    output['type'] = types.get(md.Meta.get('type', ['bug'])[0], 'bug')
    output['category'] = categories.get(md.Meta.get('category', ['unclassified'])[0], 'unclassified')
    output['validators'] = []
    output['working'] = []
    for experience in md.Meta.get('xp', ['']):
        splitted = experience.split(',')
        name = splitted[0]
        email = splitted[1]
        verified = splitted[2]
        data = {
            'name': name,
        }
        for elem in splitted[3:]:
            splitelem = elem.split('=')
            data[splitelem[0]] = splitelem[1]
        if verified == 'yes':
            output['validators'].append(data)
        elif verified == 'no':
            output['working'].append(data)
    return output

def md_parse_docs(fn):
    """
    Parse a markdown document into (section, field) keyed dict of
    docstrings.
    """

    md = Markdown(safe_mode='escape')
    with open(fn, 'r') as f_in:
        docs_html = md.convert(f_in.read())
        etr = etree.HTML(docs_html)

    docs = {}
    section = ''
    field = ''
    first_h1 = etr.find('body/h1')

    if first_h1 is not None:
        section, field = first_h1.text, ''
        for e in first_h1.itersiblings():
            if e.tag == 'h1':
                section, field = e.text, ''
            elif e.tag == 'h2':
                field = e.text
            else:
                docs[(section, field)] = docs.get((section, field), '') + etree.tostring(e)
    return docs

def load_stories(story_stuff):
    meta = {
        'title':       (True, ('one',)),
        'description': (True, ('one',)),
    }
    markdowner = Markdown(extensions=['meta'], output_format='html5')
    stories = []
    for story_file, filename in story_stuff:
        story = {}
        html = markdowner.convert(story_file.read())  # also loads metadata
        story['body'] = Markup(html)
        story['slug'] = os.path.splitext(filename)[0]

        for field, (required, filters) in meta.items():
            field_val = markdowner.Meta.get(field)
            try:
                val = apply_field_constraints(field_val, required, filters)
            except MetaError as e:
                e.apply_context(filename, field)
                raise e
            story[field] = val

        stories.append(story)
        markdowner.reset()

    return stories

class Markdown:
    def __init__(self):
        self.markdown = MarkdownParser(
                            output_format='html5',
                            extensions = ['tables'],
                        )
    def parse(self, fp):
        meta = {}
        def read_file():
            in_head = True
            for line in fp:
                if not in_head:
                    yield line
                else:
                    if len(line) < 2:
                        in_head = False
                        yield line
                    else:
                        name, value = line.split(':')
                        meta[name.lower()] = value.strip()

        
        content = self.markdown.convert(''.join(read_file()))
        self.markdown.reset()
        pagetype = meta.pop('type', 'templated') # Default Pagetype
        return pagetype, meta, content

def load_blog(blog_stuff):
    meta = {
        'title':    (True, ('one',)),
        'author':   (True, ('one',)),
        'date':     (True, ('one', 'iso-date')),
        'photo':    (False, ('one',)),
        'photo_caption': (False, (''))
    }
    markdowner = Markdown(extensions=['meta'], output_format='html5')
    posts = []
    for blog_file, filename in blog_stuff:
        post = {}
        html = markdowner.convert(blog_file.read())  # also loads metadata
        post['content'] = Markup(html)
        post['summary'] = Markup('\n'.join(html.split('\n', 1)[:-1]))
        post['slug'] = os.path.splitext(filename)[0]

        for field, (required, filters) in meta.items():
            field_val = markdowner.Meta.get(field)
            try:
                val = apply_field_constraints(field_val, required, filters)
            except MetaError as e:
                e.apply_context(filename, field)
                raise e
            post[field] = val

        posts.append(post)
        markdowner.reset()

    return posts