def normalizeURL(url):
    url = url.lower()
    url = urlparse.urldefrag(url)[0]

    # split the URL
    link_parts = urlparse.urlparse(url)

    # link has been updated, so resplitting is required
    link_parts = urlparse.urlparse(url)
    if link_parts.path == '/':
        temp = list(link_parts[:])
        temp[2] = ''
        url = urlparse.urlunparse(tuple(temp))

    # link has been updated, so resplitting is required
    link_parts = urlparse.urlparse(url)
    try:
        if link_parts.netloc.split(':')[1] == '80' or \
                        link_parts.netloc.split(':')[1] == '443':
            temp = list(link_parts[:])
            temp[1] = temp[1].split(':')[0]
            url = urlparse.urlunparse(tuple(temp))
    except IndexError:
        pass

    url = url.decode('utf-8', 'ignore')

    return url

    def __init__(self, merchant_code, secret_code, merchant_titular, merchant_name, terminal_number, return_url=None, transaction_type=None, lang=None, domain=None, domain_protocol="http", currency_code=None, redirect_url=None, **kwargs):
        self._merchant_code = merchant_code
        self._secret_code = secret_code
        self._merchant_titular = merchant_titular
        self._merchant_name = merchant_name
        self._terminal_number = terminal_number
        self._redirect_url = redirect_url

        self._lang = lang or self._lang
        self._transaction_type = transaction_type or self._transaction_type

        self._currency_code = currency_code or self._currency_code

        self._domain = domain or urlparse.urlunparse((
                    domain_protocol,
                    Site.objects.get_current().domain,
                    '/',
                    None,
                    None,
                    None))

        domain = urlparse.urlparse(self._domain)
        merchant_path = reverse('process_payment', args=[kwargs.get('variant')])
        self._merchant_url = urlparse.urlunparse((domain.scheme, domain.netloc, merchant_path, None, None, None))

        self._return_url = return_url

        return super(CaixaCatalunyaBaseProvider, self).__init__(**kwargs)

    def handle_endtag(self, tag):
        if self._link_counter is not None:
            if self._link_counter < 1:
                if tag != 'a':
                    logger.warn(
                        u'Invalid HTML tags in %s', self._url)
                href = self._get_link_attr('href')
                #  We discard anchors and empty href.
                if href and href[0] != '#':
                    href_parts = urlparse.urlparse(href)
                    # Convert absolute URL to absolute URI
                    if href[0] == '/':
                        href = urlparse.urlunparse(
                            self._base_uri +  href_parts[2:])
                    elif not is_remote_uri(href):
                        # Handle relative URL
                        href = urlparse.urlunparse(
                            self._base_uri +
                            ('/'.join((self._relative_path, href_parts[2])),) +
                            href_parts[3:])

                    filename = os.path.basename(href_parts[2])
                    # If the content of the link is empty, we use the last
                    # part of path.
                    if self._buffer:
                        name = ' '.join(self._buffer)
                    else:
                        name = filename
                    rel = self._get_link_attr('rel')
                    self.links.append((href, filename, name, rel),)
                self._link_counter = None
                self._link_attrs = None
                self._buffer = None
            else:
                self._link_counter -= 1

def form_url(parenturl,url):
     url = url.strip() # ran across an image with a space in the
                       # src. Browser handled it, so we'd better, too.
 
     if "//" in url or parenturl == None:
         returl = url
     else:
         parsedUrl = urlparse.urlparse(parenturl)
         if url.startswith("/") :
             returl = urlparse.urlunparse(
                 (parsedUrl.scheme,
                  parsedUrl.netloc,
                  url,
                  '','',''))
         else:
             toppath=""
             if parsedUrl.path.endswith("/"):
                 toppath = parsedUrl.path
             else:
                 toppath = parsedUrl.path[:parsedUrl.path.rindex('/')]
             returl = urlparse.urlunparse(
                 (parsedUrl.scheme,
                  parsedUrl.netloc,
                  toppath + '/' + url,
                  '','',''))
     return returl

def open_with_auth(url):
    """Open a urllib2 request, handling HTTP authentication"""

    scheme, netloc, path, params, query, frag = urlparse.urlparse(url)

    if scheme in ('http', 'https'):
        auth, host = urllib2.splituser(netloc)
    else:
        auth = None

    if auth:
        auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
        new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
        request = urllib2.Request(new_url)
        request.add_header("Authorization", auth)
    else:
        request = urllib2.Request(url)

    request.add_header('User-Agent', user_agent)
    fp = urllib2.urlopen(request)

    if auth:
        # Put authentication info back into request URL if same host,
        # so that links found on the page will work
        s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
        if s2==scheme and h2==host:
            fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))

    return fp

 def validate_ticket(self, ticket, request):
     service_name = self.service_name
     ticket_name = self.ticket_name
     this_url = self.get_url(request)
     p = urlparse.urlparse(this_url)
     qs_map = urlparse.parse_qs(p.query)
     if ticket_name in qs_map:
         del qs_map[ticket_name]
     param_str = urlencode(qs_map, doseq=True)
     p = urlparse.ParseResult(*tuple(p[:4] + (param_str,) + p[5:]))
     service_url = urlparse.urlunparse(p)
     params = {
             service_name: service_url,
             ticket_name: ticket,}
     param_str = urlencode(params, doseq=True)
     p = urlparse.urlparse(self.cas_info['service_validate_url'])
     p = urlparse.ParseResult(*tuple(p[:4] + (param_str,) + p[5:]))
     service_validate_url = urlparse.urlunparse(p)
     self.log(
         "Requesting service-validate URL => '{0}' ...".format(
             service_validate_url))
     http_client = HTTPClient(self.cas_agent) 
     d = http_client.get(service_validate_url)
     d.addCallback(treq.content)
     d.addCallback(self.parse_sv_results, service_url, ticket, request)
     return d

 def doHarvest(self, fromDate, until):
     lrUrl = self.config['lr']['url']
     if not fromDate:
         fromDate = self.config['lr']['first_run_start_date']
     urlParts = urlparse.urlparse(lrUrl)
     params = {"until": until}
     if fromDate:
         params['from'] = fromDate
     newQuery = urllib.urlencode(params)
     lrUrl = urlparse.urlunparse((urlParts[0],
                                  urlParts[1],
                                  '/harvest/listrecords',
                                  urlParts[3],
                                  newQuery,
                                  urlParts[5]))
     resumption_token = self.harvestData(lrUrl)
     while resumption_token is not None:
         newQuery = urllib.urlencode({"resumption_token": resumption_token})
         lrUrl = urlparse.urlunparse((urlParts[0],
                                  urlParts[1],
                                  '/harvest/listrecords',
                                  urlParts[3],
                                  newQuery,
                                  urlParts[5]))
         resumption_token = self.harvestData(lrUrl)

    def _stripSitePath(self, uri, parms):
        """
            Strip off our site-host and site-path from 'uri'.
        """
        ( scheme
        , netloc
        , path
        , url_parm
        , query
        , fragment
        ) = urlparse.urlparse( uri )

        site_host = urlparse.urlunparse( ( scheme, netloc, '', '', '', '' ) )

        if scheme and parms.get( 'site_host' ) is None:
            parms[ 'site_host' ] = site_host

        if site_host != parms[ 'site_host' ]: # XXX foreign site!  Punt!
            return None, None

        if self._site_path and path.startswith( self._site_path ):
            path = path[ len( self._site_path ) : ]

        uri = urlparse.urlunparse(
                            ( '', '', path, url_parm, query, fragment ) )

        return uri, query

def makeArchive(srcrepo, archive, pkgs):
    # pull is the pkgrecv(1) command
    import pull

    print "source directory:", srcrepo
    print "package list:", pkgs

    urlprefix = ['http://', 'https://', 'file://']
    if not True in [srcrepo.startswith(i) for i in urlprefix]:
        # We need the replace statement because the urllib.url2pathname
        # command used in pull.py will work correctly with '/' slash in
        # windows.
        srcrepo = urlunparse(("file", os.path.abspath(srcrepo).replace('\\', '/'), '','','',''))

    destrepo = tempfile.mkdtemp()
    if not True in [destrepo.startswith(i) for i in urlprefix]:
        destrepo_url = urlunparse(("file", os.path.abspath(destrepo).replace('\\', '/'), '','','',''))

    sys.argv = [sys.argv[0], '-m', 'all-timestamps', '-s', srcrepo, '-d', destrepo_url]
    sys.argv.extend(pkgs)
    rv = pull.main_func()

    #copy the cfg_cache to the archive
    if isinstance(archive, zipfile.ZipFile):
        for root, dirs, files in os.walk(destrepo, topdown=False):
            reldir = root[len(destrepo)+1:]
            for name in files:
                archive.write(os.path.join(root, name), os.path.join(reldir, name))
    elif isinstance(archive, tarfile.TarFile):
        archive.add(destrepo, destrepo[len(destrepo):])

    #close the archive
    archive.close()
    return rv

 def get_download_links(self, folderid=None, fields=""):
     download_links = dict()
     download_api_url = self.base_url[:]
     fields = (set(fields.split(",")) | set(["filename"])) - set([""])
     if len(fields) > 1:
         supported_fields = set("downloads,lastdownload,filename,size,killcode,serverid,type,x,y,realfolder,bodtype,killdeadline,licids,uploadtime".split(","))
         for field in fields:
             if field not in supported_fields:
                 raise HosterAPIError, "field: %s not supported" % field
     query = urllib.urlencode({"sub": "listfiles",
                               "fields": ",".join(fields)})
     download_api_url[4] += "&"+query
     if folderid is not None:
         query = urllib.urlencode({"realfolder": folderid})
         download_api_url[4] += "&"+query
     url = urlunparse(download_api_url)
     lines = urllib2.urlopen(url).readlines()
     self._catch_error(lines, url)
     for line in lines:
         try:
             rows = line.split(",")
             fileid = rows[0]
             properties = dict(zip(fields, rows[1:]))
             properties["filename"] = properties["filename"].replace("\n", "")
             download_url = [self.scheme, "rapidshare.com", "files/%s/%s" % (fileid, properties["filename"]), "", "", ""]
             properties["url"] = urlunparse(download_url)
             download_links[fileid] = properties
         except (ValueError, KeyError):
             pass
     return download_links

def translate_git_url(git_url, commit_id):
    """Create a real git URL based on defined translations.

    :param git_url: The git URL as obtained from the backend.
    :param commit_id: The git SHA.
    :return The base URL to create URLs, and the real commit URL.
    """

    base_url = None
    commit_url = None

    if git_url and commit_id:
        t_url = urlparse.urlparse(git_url)
        known_git_urls = CONFIG_GET("KNOWN_GIT_URLS")

        if t_url.netloc in known_git_urls.keys():
            known_git = known_git_urls.get(t_url.netloc)

            path = t_url.path
            for replace_rule in known_git[3]:
                path = path.replace(*replace_rule)

            base_url = urlparse.urlunparse((
                known_git[0], t_url.netloc, known_git[1] % path,
                "", "", ""
            ))
            commit_url = urlparse.urlunparse((
                known_git[0], t_url.netloc,
                (known_git[2] % path) + commit_id,
                "", "", ""
            ))

    return base_url, commit_url

def download_css(soup, url_parts, dst_folder, index_path):
    """ 
    parse css src's and download css to dst_folder 
    """
    tmp_url_parts = deepcopy(url_parts)
    for css in soup.findAll("link", {"rel": "stylesheet"}):
        if css.has_key("href"):
            file_name = css["href"].split("/")[-1]
            file_name = sanitize_file_name(file_name)

            logging.debug("Downloading css " + file_name + "...")

            new_src = create_directories(dst_folder, list(urlparse.urlparse(css["href"]))[2])
            full_path = os.path.join(dst_folder, new_src)
            outpath = os.path.join(full_path, file_name)

            if css["href"].lower().startswith("http"):
                tmp_url_parts = list(urlparse.urlparse(css["href"]))
                download_file(css["href"], outpath)
            else:
                tmp_url_parts[2] = css["href"]
                download_file(urlparse.urlunparse(tmp_url_parts), outpath)

            root_url = urlparse.urlunparse(tmp_url_parts)
            file_name_index = root_url.index(file_name)
            root_url = root_url[:file_name_index]
            download_css_imports(
                soup, list(urlparse.urlparse(css["href"])), root_url, new_src + "/" + file_name, dst_folder, index_path
            )
            css["href"] = css["href"].replace(css["href"], index_path + "/" + new_src + "/" + file_name)

def _add_utm_param(url, type, sourse, campaign, name, matching):
    url_parts = list(urlparse.urlparse(url))
    if not _url_param_safe_check(url_parts[4]):
        return urlparse.urlunparse(url_parts)
    query = dict(urlparse.parse_qsl(url_parts[4]))

    if (type == 'banner'):
        utm_medium = 'cpm_yottos'
    else:
        utm_medium = 'cpc_yottos'
    utm_source =  str(sourse or 'other')
    utm_campaign = str(campaign)
    utm_content = str(name)
    if query.has_key('utm_source'):
        utm_term = str(sourse or 'other')
    else:
        utm_term = str(matching)

    if not query.has_key('utm_medium'):
        query.update({'utm_medium':utm_medium})
    if not query.has_key('utm_source'):
        query.update({'utm_source':utm_source})
    if not query.has_key('utm_campaign'):
        query.update({'utm_campaign':utm_campaign})
    if not query.has_key('utm_content'):
        query.update({'utm_content':utm_content})
    if not query.has_key('utm_term'):
        query.update({'utm_term':utm_term})
    url_parts[4] = urllib.urlencode(query)
    return urlparse.urlunparse(url_parts)

    def _secure_request(self, url, method, data=None, files=None, headers=None,
                        raw=False, send_as_json=True, content_type=None,
                        **request_kwargs):

        full_url = self.build_url(url)

        # Add token (if it's not already there)
        if self._token:
            parsed = list(urlparse.urlparse(full_url))
            if not parsed[4]:  # query
                parsed[4] = 'token=%s' % self._token
                full_url = urlparse.urlunparse(parsed)
            elif 'token' not in urlparse.parse_qs(parsed[4]):
                parsed[4] += '&token=%s' % self._token
                full_url = urlparse.urlunparse(parsed)
        headers = headers or {}

        # If files are being sent, we cannot encode data as JSON
        if send_as_json and not files:
            headers['content-type'] = 'application/json'
            data = json.dumps(data or {})
        else:
            if content_type:
                headers['content-type'] = content_type
            data = data or ''

        method = getattr(requests, method, None)
        response = method(full_url, data=data, files=files, headers=headers,
                          **request_kwargs)
        self.check_for_errors(response)  # Raise exception if something failed

        if raw or not response.content:
            return response.content
        return json.loads(response.text)

def format_message(template, config, first_name=None, last_name=None, uid=None, target_email=None):
	first_name = ('Alice' if not isinstance(first_name, (str, unicode)) else first_name)
	last_name = ('Liddle' if not isinstance(last_name, (str, unicode)) else last_name)
	target_email = ('[email protected]' if not isinstance(target_email, (str, unicode)) else target_email)
	uid = (uid or config['server_config'].get('server.secret_id') or make_uid())

	template = template_environment.from_string(template)
	template_vars = {}
	template_vars['uid'] = uid
	template_vars['first_name'] = first_name
	template_vars['last_name'] = last_name
	template_vars['email_address'] = target_email
	template_vars['company_name'] = config.get('mailer.company_name', '')

	webserver_url = config.get('mailer.webserver_url', '')
	webserver_url = urlparse.urlparse(webserver_url)
	tracking_image = config['server_config']['server.tracking_image']
	template_vars['webserver'] = webserver_url.netloc
	tracking_url = urlparse.urlunparse((webserver_url.scheme, webserver_url.netloc, tracking_image, '', 'id=' + uid, ''))
	webserver_url = urlparse.urlunparse((webserver_url.scheme, webserver_url.netloc, webserver_url.path, '', '', ''))
	template_vars['tracking_dot_image_tag'] = "<img src=\"{0}\" style=\"display:none\" />".format(tracking_url)

	template_vars_url = {}
	template_vars_url['rickroll'] = 'http://www.youtube.com/watch?v=oHg5SJYRHA0'
	template_vars_url['webserver'] = webserver_url + '?id=' + uid
	template_vars_url['webserver_raw'] = webserver_url
	template_vars_url['tracking_dot'] = tracking_url
	template_vars['url'] = template_vars_url
	template_vars.update(template_environment.standard_variables)
	return template.render(template_vars)

    def __init__(self, scheme=None, hostname=None, path="/",
                 params="", query={}, fragment="",
                 username=None, password=None, port=None):

        self.path = path
        self.params = params
        self.query = query
        self.fragment = fragment

        if hostname:
            # Absolute URL
            if username:
                if password:
                    netloc = username + ':' + password + '@' + hostname
                else:
                    netloc = username + '@' + hostname
            else:
                netloc = hostname

            if not scheme:
                scheme = DEFAULT_SCHEME

            host = hostname

            defport = DEFAULT_PORTS.get(scheme, None)

            if port:
                if port != defport:
                    netloc = netloc + ':' + str(port)
                    host = host + ':' + str(port)
            else:
                port = defport

            self.scheme = scheme
            self.netloc = netloc
            self.host = host
            self.hostname = hostname
            self.username = username
            self.password = password
            self.port = port

        else:
            # Relative URL
            self.scheme = ""
            self.netloc = ""
            self.host = ""
            self.hostname = ""
            self.username = None
            self.password = None
            self.port = None

        query_string = unparse_qs(self.query)
        quoted_path = urllib.quote(self.path)

        self.location = urlparse.urlunparse(('', '', quoted_path, self.params,
                                             query_string, self.fragment))

        self.url = urlparse.urlunparse((self.scheme, self.netloc, quoted_path,
                                        self.params, query_string,
                                        self.fragment))

    def setUp(self):
        super(TestShotgunDownloadAndUnpack, self).setUp()

        zip_file_location = os.path.join(self.fixtures_root, "misc", "zip")
        # Identify the source file to "download"
        self.download_source = os.path.join(zip_file_location, "tank_core.zip")
        # store the expected contents of the zip, to ensure it's properly
        # extracted.
        self.expected_output_txt = os.path.join(zip_file_location, "tank_core.txt")
        self.expected_output = open(self.expected_output_txt).read().split("\n")

        # Construct URLs from the source file name
        # "file" will be used for the protocol, so this URL will look like
        # `file:///fixtures_root/misc/zip/tank_core.zip`
        self.good_zip_url = urlparse.urlunparse(
            ("file", None, self.download_source, None, None, None)
        )
        self.bad_zip_url = urlparse.urlunparse(
            ("file", None, self.download_source, None, None, None)
        )

        # Temporary destination to unpack sources to.
        self.download_destination = os.path.join(
            self.tank_temp, self.short_test_name, "test_unpack"
        )
        os.makedirs(os.path.dirname(self.download_destination))
        if os.path.exists(self.download_destination):
            os.remove(self.download_destination)

        # Make sure mockgun is properly configured
        if self.mockgun.config.server is None:
            self.mockgun.config.server = "unit_test_mock_sg"

    def _starter1(self):
        # to be honest, i ignore why I should make this GET request...
        headers = dict(self.headers)
        headers.update({
            "Referer":
            urlunparse((__PROTOCOL__,
                        self.host,
                        __STARTER_PATH__, None, None, None))})

        url = urlunparse((__PROTOCOL__,
                          self.host,
                          __STARTER_PATH__, None, None, None))
        log.debug(url)
        cookies = dict(
            DSSignInURL="/",
            DSLastAccess=self.DSLastAccess,
            DSID=self.DSID,
            DSFirstAccess=self.last_res.cookies['DSLastAccess'])

        params = dict(check="yes")

        res = requests.get(
            url,
            headers=headers,
            params=params,
            cookies=cookies,
            verify=False)
        log.debug(res.text)
        res.raise_for_status()

def canonURL(url, parent_domain):

    (scheme, netloc, path, parameters, query, fragment) = urlparse(url)

    parent = urlparse(parent_domain)

    if not netloc and not path:
        return ""

    if not netloc and path.startswith("."):
        new_url = urljoin(parent_domain, path)
        (scheme, netloc, path, parameters, query, fragment) = urlunparse(new_url)

    elif not netloc and path:
        netloc = parent.netloc

    netloc_lower = netloc.lower()
    netloc = netloc_lower.split(":")[0]

    prev_path = path
    print prev_path
    while 1:
        path = collapse_url.sub('/', path, 1)
        print path
        if prev_path == path:
            break
        prev_path = path

    path = unquote(path)

    canon_url = urlunparse((scheme, netloc, path, "", "", ""))
    return canon_url

def get_favicon(url):
    try:
        html = requests.request('GET', url)
        soup = BeautifulSoup(html.text)
        icon = soup.find('link', rel='shortcut icon')
        if icon is None:
            icon = soup.find('link', type='image/x-icon')
        icon_href = None
        if hasattr(icon, 'href'):
            icon_href = str(icon['href'])
        if icon_href is None or icon_href.strip() == "":
            parsed_url = urlparse.urlparse(url)
            icon_href = urlparse.urlunparse((parsed_url.scheme, parsed_url.netloc, '', '', '', '')) + '/favicon.ico'
            last_try = requests.request('GET', icon_href)
            if last_try.status_code == 200:
                return icon_href
            else:
                return None
        if "http://" not in icon_href:
            parsed_url = urlparse.urlparse(url)
            icon_href = urlparse.urlunparse((parsed_url.scheme, parsed_url.netloc, '', '', '', '')) + icon_href
        last_try = requests.request('GET', icon_href)
        if last_try.status_code == 200:
            return icon_href
        else:
            return 'https://www.readbox.co/static/lightpng.png'
    except Exception:
        return 'https://www.readbox.co/static/lightpng.png'