def get(self, url, default=NO_DEFAULT, headers=None):
     """Retrieve a Response object for a given URL.
     """
     headers = headers or {}
     url = URL(url)
     row = self.db.execute(
         select([
             responses.c.created,
             responses.c.host,
             responses.c.request_url,
             responses.c.accept,
             responses.c.url,
             responses.c.headers,
             responses.c.content])
         .where(and_(
             responses.c.request_url == url.as_string(),
             responses.c.accept == headers.get('Accept', '')))
     ).fetchone()
     if not row:
         log.info('cache miss %s' % url)
         row = self.add(url, headers)
         if row is None:
             if default is NO_DEFAULT:
                 raise KeyError(url)
             log.info('invalid url %s' % url)
             return default
     else:
         log.info('cache hit %s' % url)
     return Response(*row)

class Segments(object):
    def __init__(self, base, path, segments, defaults):
        self.base = PURL(base, path=path)
        self.segments = OrderedDict(zip(segments, defaults))

    def build(self):
        segments = self.base.path_segments() + tuple(self.segments.values())
        url = self.base.path_segments(segments)
        return url

    def __str__(self):
        return self.build().as_string()

    def _get_segment(self, segment):
        return self.segments[segment]

    def _set_segment(self, segment, value):
        self.segments[segment] = value

    @classmethod
    def _segment(cls, segment):
        return property(
                fget=lambda x: cls._get_segment(x, segment),
                fset=lambda x, v: cls._set_segment(x, segment, v),
                )

    def __call__(self, request, extra_classes=None):
        """ Renders the element. """

        a = builder.A(request.translate(self.text))

        if self.request_method == 'GET':
            a.attrib['href'] = self.url

        if self.request_method == 'DELETE':
            url = URL(self.url).query_param(
                'csrf-token', request.new_csrf_token())

            a.attrib['ic-delete-from'] = url.as_string()

        if self.classes or extra_classes:
            classes = self.classes + (extra_classes or tuple())
            a.attrib['class'] = ' '.join(classes)

        # add the hidden from public hint if needed
        if self.is_hidden_from_public:

            # This snippet is duplicated in the hidden-from-public-hint macro!
            hint = builder.I()
            hint.attrib['class'] = 'hidden-from-public-hint'
            hint.attrib['title'] = request.translate(
                _("This site is hidden from the general public")
            )

            a.append(builder.I(' '))
            a.append(hint)

        for key, value in self.attributes.items():
            a.attrib[key] = request.translate(value)

        return tostring(a)

    def editbar_links(self):
        if self.request.is_logged_in:
            edit_url = URL(self.request.link(self.model.event, 'bearbeiten'))
            edit_url = edit_url.query_param(
                'return-to', self.request.link(self.model.event)
            )
            edit_link = Link(
                text=_("Edit"),
                url=edit_url.as_string(),
                classes=('edit-link', )
            )

            if self.event_deletable(self.model.event):
                delete_link = DeleteLink(
                    text=_("Delete"),
                    url=self.request.link(self.model.event),
                    confirm=_("Do you really want to delete this event?"),
                    yes_button_text=_("Delete event"),
                    redirect_after=self.events_url
                )
            else:
                delete_link = DeleteLink(
                    text=_("Delete"),
                    url=self.request.link(self.model.event),
                    confirm=_("This event can't be deleted."),
                    extra_information=_(
                        "To remove this event, go to the ticket and reject it."
                    )
                )

            return [edit_link, delete_link]

 def select(self, url):
     print "SELECTED!!"
     url = URL(url)
     field_name = '%s_exact' % self.facet_name
     url = url.append_query_param('selected_facets', '%s:%s' % (field_name, self.key))
     self.selected = True
     return url

    def get_links(self, request):

        edit_link = URL(request.link(self.submission))
        edit_link = edit_link.query_param("edit", "")
        edit_link = edit_link.query_param("return-to", request.url)

        return [Link(text=_("Edit submission"), url=edit_link.as_string(), classes=("edit-link",))]

def get_wikipedia(args, name):
    url = URL(
        "http://en.wikipedia.org/w/api.php?format=json&action=query&prop=info&inprop=url")

    def get_result(res):
        """
{
    "query": {
        "pages": {
            "45724": {
                "contentmodel": "wikitext",
                "counter": "",
                "editurl":
                    "http://en.wikipedia.org/w/index.php?title=Panthera_leo&action=edit",
                "fullurl": "http://en.wikipedia.org/wiki/Panthera_leo",
                "lastrevid": 535861485,
                "length": 71,
                "ns": 0,
                "pageid": 45724,
                "pagelanguage": "en",
                "redirect": "",
                "title": "Panthera leo",
                "touched": "2014-02-27T02:04:39Z"
            }
        }
    }
}
        """
        res = res.json().get('query', {}).get('pages', {})
        return res.values()[0] if res else {}

    return get_data(args, name, 'wikipedia', url.query_param('titles', name), get_result)

def get_bib(args):
    uploaded = load(args.data_file('repos', 'cdstar.json'))
    fname_to_cdstar = {}
    for type_ in ['texts', 'docs', 'data']:
        for hash_, paths in load(args.data_file('repos', type_ + '.json')).items():
            if hash_ in uploaded:
                for path in paths:
                    fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_]
    for hash_, paths in load(args.data_file('repos', 'edmond.json')).items():
        if hash_ in uploaded:
            for path in paths:
                fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_]
    db = Database.from_file(args.data_file('repos', 'Dogon.bib'), lowercase=True)
    for rec in db:
        doc = Document(rec)
        newurls = []
        for url in rec.get('url', '').split(';'):
            if not url.strip():
                continue
            if url.endswith('sequence=1'):
                newurls.append(url)
                continue
            url = URL(url.strip())
            if url.host() in ['dogonlanguages.org', 'github.com', '']:
                fname = url.path().split('/')[-1]
                doc.files.append((fname, fname_to_cdstar[fname]))
            else:
                newurls.append(url.as_string())
        doc.rec['url'] = '; '.join(newurls)
        yield doc

    def __init__(self, url, save_dir='tmp'):
        """
        @url: full url of a site
        @save_dir: dir to save site
        """
        # log
        self.logger = logger('file', 'sitelog.log', save_dir)
        self.logger.info('-' * 20)
        self.logger.info('start')
        self.logger.info('start func: __init__')
        self.logger.info('url: %s' % url)

        save_time = datetime.strftime(datetime.now(), '%Y%m%d%H%M')
        self.save_time = save_time
        self.save_dir = os.path.abspath(os.path.join(save_dir, save_time))
        # create dir if not exist
        if not os.path.isdir(self.save_dir):
            os.makedirs(self.save_dir)

        self.url = url
        u = URL(url)
        # get host like: http://m.sohu.xom
        self.host = u.scheme() + '://' + u.host()
        print '%s: saving %s' % (save_time, self.url)
        self.logger.info('end func: __init__')

def cc_link(req, license_url, button="regular"):
    if license_url == "http://en.wikipedia.org/wiki/Public_domain":
        license_url = "http://creativecommons.org/publicdomain/zero/1.0/"
    license_url = URL(license_url)
    if license_url.host() != "creativecommons.org":
        return

    comps = license_url.path().split("/")
    if len(comps) < 3:
        return  # pragma: no cover

    known = {
        "zero": "Public Domain",
        "by": "Creative Commons Attribution License",
        "by-nc": "Creative Commons Attribution-NonCommercial License",
        "by-nc-nd": "Creative Commons Attribution-NonCommercial-NoDerivatives License",
        "by-nc-sa": "Creative Commons Attribution-NonCommercial-ShareAlike License",
        "by-nd": "Creative Commons Attribution-NoDerivatives License",
        "by-sa": "Creative Commons Attribution-ShareAlike License",
    }
    if comps[2] not in known:
        return

    icon = "cc-" + comps[2] + ("-small" if button == "small" else "") + ".png"
    img_attrs = dict(alt=known[comps[2]], src=req.static_url("clld:web/static/images/" + icon))
    height, width = (15, 80) if button == "small" else (30, 86)
    img_attrs.update(height=height, width=width)
    return HTML.a(HTML.img(**img_attrs), href=license_url, rel="license")

def cc_link(req, license_url, button='regular'):
    if license_url == 'https://en.wikipedia.org/wiki/Public_domain':
        license_url = 'https://creativecommons.org/publicdomain/zero/1.0/'
    license_url = URL(license_url)
    if license_url.host() != 'creativecommons.org':
        return

    comps = license_url.path().split('/')
    if len(comps) < 3:
        return  # pragma: no cover

    known = {
        'zero': 'Public Domain',
        'by': 'Creative Commons Attribution License',
        'by-nc': 'Creative Commons Attribution-NonCommercial License',
        'by-nc-nd': 'Creative Commons Attribution-NonCommercial-NoDerivatives License',
        'by-nc-sa': 'Creative Commons Attribution-NonCommercial-ShareAlike License',
        'by-nd': 'Creative Commons Attribution-NoDerivatives License',
        'by-sa': 'Creative Commons Attribution-ShareAlike License'}
    if comps[2] not in known:
        return

    icon = 'cc-' + comps[2] + ('-small' if button == 'small' else '') + '.png'
    img_attrs = dict(
        alt=known[comps[2]],
        src=req.static_url('clld:web/static/images/' + icon))
    height, width = (15, 80) if button == 'small' else (30, 86)
    img_attrs.update(height=height, width=width)
    return HTML.a(HTML.img(**img_attrs), href=license_url, rel='license')

def set_param(request=None, url=None, **kwargs):
    if not request and not url:
        return '/'
    url = URL(path=request.path, query=request.META['QUERY_STRING']) if request else URL(url)
    for k, v in kwargs.items():
        url = url.query_param(k, v)
    return url.as_string()

    def activate(self, *args, **kwargs):
        """
        Transition: inactive -> active

        Set github webhook and activate all Presentations.
        """
        try:
            provider = self.user.social_auth.get(provider="github")
        except UserSocialAuth.DoesNotExists:
            logger.error("No social auth provider for Github found on user")
            return
        github = Github(provider.access_token)
        try:
            repo = github.get_user().get_repo(self.name)
        except GithubException:
            logger.error("Could not find repository")
            return
        url = URL(
            scheme="https",
            host=self.site.domain,
            path=reverse("qraz:webhook", kwargs={"username": self.user.username, "repository": self.name}),
        )
        try:
            hook = repo.create_hook(
                "web",
                {"url": url.as_string(), "content_type": "json", "secret": self.secret},
                events=["push"],
                active=True,
            )
        except GithubException as excp:
            logger.error("Could not create webhook: %s", excp)
            return
        self.hook = hook.id

def get_data(q_link):
    url = URL(q_link)
    if url.domain() not in ['quora.com', 'www.quora.com']:
        return 'error, not quora'

    url = URL(
        scheme='https',
        host='www.quora.com',
        path=url.path(),
        query='share=1').as_string()

    soup = BeautifulSoup(requests.get(url).text)

    question = {}
    question['url'] = url
    question['title'] = soup.find("div", {"class": "question_text_edit"}).text
    question['topics'] = [topic.text for topic in soup.find_all("div", {"class": "topic_list_item"})]
    question['details'] = soup.find("div", {"class": "question_details_text"}).text

    answers = []

    divs = soup.find_all("div", {"class": "pagedlist_item"})
    
    try:
        ans_count = soup.find("div", {"class": "answer_header_text"}).text.strip()
        count = int(re.match(r'(\d+) Answers', ans_count).groups()[0])
    except:
        return jsonify(question=question, answers=answers)

    question['answer_count'] = count

    count = len(divs) - 1 if count < 6 else 6
    for i in range(count):
        one_answer = {
            'votes': '-1',
            'rank': 0,
            'answer': ''
        }
        try:
            author = {}
            author['name'] = divs[i].find("div", {"class": "answer_user"}).find("span", {"class": "answer_user_wrapper"}).find("a", {"class": "user"}).string
            author['bio'] = divs[i].find("div", {"class": "answer_user"}).find("span", {"class": "answer_user_wrapper"}).find_all("span", {"class": "rep"})[1].find("span", {"class": "hidden"}).text
        except:
            author['name'] = 'Anonymous'
            author['bio'] = ''
        one_answer['author'] = author

        one_answer['votes'] = divs[i].find("span", {"class":"numbers"}).text

        html_block = divs[i].find("div", {"id": re.compile("(.*)_container")}).contents
        answer_html = ''
        for p in range(len(html_block) - 1):
            answer_html += str(html_block[p])
        one_answer['answer_html'] = answer_html
        one_answer['answer'] = divs[i].find("div", {"class": "answer_content"}).text
        one_answer['rank'] = i + 1
        answers.append(one_answer)

    return jsonify(question=question, answers=answers)

    def info(self, url):
        """Interface method to be called when processing new images.

        This method ties together the DataProvider workflow.
        """
        url = URL(url)
        return self.postprocess(
            self.info_for_id(self.id_from_url(url, url.host(), url.path_segments())))

 def setUp(self):
     self.unicode_param = 'значение'
     # Python 2.6 requires bytes for quote
     self.urlencoded_param = quote(self.unicode_param.encode('utf8'))
     url = 'http://www.google.com/blog/article/1?q=' + self.urlencoded_param
     self.ascii_url = URL.from_string(url.encode('ascii'))
     # django request.get_full_path() returns url as unicode
     self.unicode_url = URL.from_string(url)

 def url(self):
     kwargs = {
         "username": self.repository.user.username,
         "repository": self.repository.name,
         "presentation": self.name,
     }
     url = URL(scheme="http", host=self.repository.site.domain, path=reverse("qraz:download", kwargs=kwargs))
     return url.as_string()

def maybe_license_link(req, license, **kw):
    cc_link_ = cc_link(req, license, button=kw.pop('button', 'regular'))
    if cc_link_:
        return cc_link_
    license_url = URL(license)
    if license_url.host():
        return external_link(license_url, **kw)
    return license

def cached_path(args, url, rel):
    if url.startswith('/'):
        return url
    url_ = URL(url)
    cached = args.data_file('edmond', url_.path_segments()[-1])
    if not cached.exists():
        fname, headers = urlretrieve(url, '%s' % cached)
    return str(cached)

def append_path(url, url_path):
    target = URL(url_path)
    if url_path.startswith('/'):
        url = url.path(target.path())
    else:
        url = url.add_path_segment(target.path())
    if target.query():
        url = url.query(target.query())
    return url.as_string()