def test(rules, example, pld, tld):
     try:
         url.set_psl(rules)
         assert_equal(url.parse(example).pld, pld)
         assert_equal(url.parse(example).tld, tld)
     finally:
         url.set_psl(pkgutil.get_data('url', 'psl/2016-08-16.psl'))

 def __init__(self, target, **params):
     if isinstance(target, basestring):
         self._host = url.parse(target)
     elif isinstance(target, (tuple, list)):
         self._host = url.parse('http://%s:%s/' % target)
     else:
         raise TypeError('Host must be a string or tuple')
     self._params = params

    def test_str_repr(self):
        '''Make sure str and repr produce reasonable results'''
        examples = [
            ('http://foo.com/', 'http://foo.com/'),
            ('http://FOO.com/', 'http://foo.com/')
        ]

        for toparse, strng in examples:
            self.assertEqual(str(url.parse(toparse)), strng)
            self.assertEqual(repr(url.parse(toparse)),
                '<url.URL object "%s" >' % strng)

 def test_empty_hostname(self):
     '''Allow empty hostnames'''
     examples = [
         'http:///path',
         'http://[email protected]/path',
         'http://:80/path',
     ]
     for example in examples:
         # Equal to itself
         self.assertEqual(url.parse(example), example)
         # String representation equal to the provided example
         self.assertEqual(url.parse(example).utf8(), example)

 def test(uni, puny, upuny, epuny):
     assert_equal(url.parse(uni).escape().punycode().utf8(), epuny)
     # Also make sure punycode is idempotent
     assert_equal(
         url.parse(uni).escape().punycode().punycode().utf8(), epuny)
     # Make sure that we can reverse the procedure correctly
     assert_equal(
         url.parse(uni).escape().punycode().unpunycode().unescape(),
         uni)
     # And we get what we'd expect going the opposite direction
     assert_equal(
         url.parse(puny).unescape().unpunycode().unicode(), uni)

def _response_to_features(response):
    features = set()
    tree = etree.HTML(response.text)

    for item in tree.iter(tag=etree.Element):
        features.add("tag-%s" % item.tag)

        if "class" in item.attrib and item.attrib["class"].strip():
            classes = whitespace.split(item.attrib["class"])
            for _c in classes:
                c = _c.strip()
                if c:
                    features.add("class-%s" % c)

        if "id" in item.attrib:
            features.add("id-%s" % item.attrib["id"])

    # path parts
    u = moz_url.parse(response.url)
    path = u._path.split("/")[1:]
    for idx, part in enumerate(path):
        features.add("path-%s-%s" % (idx, path))

    if u._query:
        for k, vl in urlparse.parse_qs(u._query).iteritems():
            features.add("qse-%s" % k)
            for v in vl:
                features.add("qsv-%s-%s" % (k, v))

    return features

    def test_abspath(self):
        '''Make sure absolute path checking works correctly'''
        examples = [
            ('howdy'           , 'howdy'        ),
            ('hello//how//are' , 'hello/how/are'),
            ('hello/../how/are', 'how/are'      ),
            ('hello//..//how/' , 'how/'         ),
            ('a/b/../../c'     , 'c'            ),
            ('../../../c'      , 'c'            ),
            ('./hello'         , 'hello'        ),
            ('./././hello'     , 'hello'        ),
            ('a/b/c/'          , 'a/b/c/'       ),
            ('a/b/c/..'        , 'a/b/'         ),
            ('a/b/.'           , 'a/b/'         ),
            ('a/b/./././'      , 'a/b/'         ),
            ('a/b/../'         , 'a/'           ),
            ('.'               , ''             ),
            ('../../..'        , ''             ),
            ('////foo'         , 'foo'          )
        ]

        base = 'http://testing.com/'
        for bad, good in examples:
            bad = base + bad
            good = base + good
            self.assertEqual(url.parse(bad).abspath().utf8(), good)

 def test_deuserinfo(self):
     '''Correctly removes userinfo'''
     examples = [
         ('http://user:[email protected]/', 'http://foo.com/'),
         ('http://[email protected]/', 'http://foo.com/')
     ]
     for bad, good in examples:
         self.assertEqual(url.parse(bad).deuserinfo().utf8(), good)

def save(url_, path="", wait=60):
    if hasattr(url_, "url"):
        url_ = url_.url
    if len(path) < 5 or "." not in path[-5:-3]:
        file = url.parse(str(url_)).filename
        path = os.path.join(path, file)
    open(path, "w").write(download(url_, wait))
    return path

 def test_tld(self):
     '''Test the pay-level domain functionality'''
     examples = [
         ('http://foo.com/bar'    , 'com'),
         ('http://bar.foo.com/bar', 'com'),
         ('/foo'                  , '')
     ]
     for query, result in examples:
         self.assertEqual(url.parse(query).tld(), result)

    def test_escape(self):
        '''Make sure we escape paths correctly'''
        examples = [
            ('hello%20and%20how%20are%20you', 'hello%20and%20how%20are%20you'),
            ('danny\'s pub'                 , 'danny%27s%20pub'              ),
            ('danny%27s pub?foo=bar&yo'     , 'danny%27s%20pub?foo=bar&yo'   ),
            # Thanks to @myronmarston for these test cases
            ('foo?bar none=foo bar'         , 'foo?bar%20none=foo%20bar'     ),
            ('foo;a=1;b=2?a=1&b=2'          , 'foo;a=1;b=2?a=1&b=2'          ),
            ('foo?bar=["hello","howdy"]'    ,
                'foo?bar=%5B%22hello%22,%22howdy%22%5D'),
        ]

        base = 'http://testing.com/'
        for bad, good in examples:
            bad = base + bad
            good = base + good
            self.assertEqual(url.parse(bad).escape().utf8(), good)
            # Escaping should also be idempotent
            self.assertEqual(url.parse(bad).escape().escape().utf8(), good)

 def test_userinfo(self):
     '''Allow a userinfo section'''
     examples = [
         ('http://user:[email protected]',   'http://user:[email protected]'),
         ('http://[email protected]', 'http://[email protected]')
     ]
     suffix = '/page.html'
     for bad, good in examples:
         bad = bad + suffix
         good = good + suffix
         self.assertEqual(url.parse(bad).utf8(), good)

    def test_absolute(self):
        '''Can it recognize if it's a relative or absolute url?'''
        examples = [
            ('http://foo.com/bar', True ),
            ('foo/'              , False),
            ('http://foo.com'    , True ),
            ('/foo/bar/../'      , False)
        ]

        for query, result in examples:
            self.assertEqual(url.parse(query).absolute(), result)

    def test_sanitize(self):
        '''Make sure the sanitize method does all that it should'''
        examples = [
            ('../foo/bar none', 'foo/bar%20none')
        ]

        base = 'http://testing.com/'
        for bad, good in examples:
            bad = base + bad
            good = base + good
            self.assertEqual(url.parse(bad).sanitize().utf8(), good)

 def test_lower(self):
     '''Can lowercase the domain name correctly'''
     examples = [
         ('www.TESTING.coM'    , 'www.testing.com/'   ),
         ('WWW.testing.com'    , 'www.testing.com/'   ),
         ('WWW.testing.com/FOO', 'www.testing.com/FOO')
     ]
     for bad, good in examples:
         bad = 'http://' + bad
         good = 'http://' + good
         self.assertEqual(url.parse(bad).utf8(), good)

    def test_defrag(self):
        '''Correctly defrags urls'''
        examples = [
            ('foo#bar', 'foo')
        ]

        base = 'http://testing.com/'
        for bad, good in examples:
            bad = base + bad
            good = base + good
            self.assertEqual(url.parse(bad).defrag().utf8(), good)

    def test_canonical(self):
        '''Correctly canonicalizes urls'''
        examples = [
            ('?b=2&a=1&c=3', '?a=1&b=2&c=3'),
            (';b=2;a=1;c=3', ';a=1;b=2;c=3')
        ]

        base = 'http://testing.com/'
        for bad, good in examples:
            bad = base + bad
            good = base + good
            self.assertEqual(url.parse(bad).canonical().utf8(), good)

def url_host_domain(url):
    """
    Return a tuple of the (host, domain) of a URL or None. Assumes that the
    URL has a scheme.
    """
    parsed = urlpy.parse(url)
    host = parsed._host
    if not host:
        return None, None
    host = host.lower()
    domain = parsed.pld().lower()
    return host, domain

    def test_punycode(self):
        '''Make sure punycode encoding works correctly'''
        examples = [
            (u'http://www.kündigen.de/',
                'http://www.xn--kndigen-n2a.de/'),
            (u'http://россия.иком.museum/',
                'http://xn--h1alffa9f.xn--h1aegh.museum/'),
            (u'http://россия.иком.museum/испытание.html',
                'http://xn--h1alffa9f.xn--h1aegh.museum/%D0%B8%D1%81%D0%BF%D1%8B%D1%82%D0%B0%D0%BD%D0%B8%D0%B5.html')
        ]

        for uni, puny in examples:
            self.assertEqual(url.parse(uni).escape().punycode().utf8(), puny)
            # Also make sure punycode is idempotent
            self.assertEqual(
                url.parse(uni).escape().punycode().punycode().utf8(), puny)
            # Make sure that we can reverse the procedure correctly
            self.assertEqual(
                url.parse(uni).escape().punycode().unpunycode().unescape(),
                uni)
            # And we get what we'd expect going the opposite direction
            self.assertEqual(
                url.parse(puny).unescape().unpunycode().unicode(), uni)

        # Make sure that we can't punycode or unpunycode relative urls
        examples = ['foo', '../foo', '/bar/foo']
        for relative in examples:
            self.assertRaises(TypeError, url.parse(relative).punycode)
            self.assertRaises(TypeError, url.parse(relative).unpunycode)

def normalize(url):
    """ Uses the Moz URL library to normalise and strip the URLs of
        extraneous information, and the urlparse library to ensure it
        is not a blank URL.
    """

    if url[:4] != 'http':
        url = 'http://'+url
    url = url.lower()
    url_parts = urlparse(url)
    if url_parts.netloc:
        url_obj = parse(url).defrag().abspath().canonical().punycode()
        return url_obj.utf8()