def open_with_auth2(url):
    """
    Open a urllib2 request, handling HTTP authentication
    In this version, user-agent is ignored
    """

    scheme, netloc, path, params, query, frag = urlparse(url)

    if scheme in ('http', 'https'):
        auth, host = splituser(netloc)
    else:
        auth = None

    if auth:
        auth = "Basic " + unquote(auth).encode('base64').strip()
        new_url = urlunparse((scheme, host, path, params, query, frag))
        request = Request(new_url)
        request.add_header("Authorization", auth)
    else:
        request = Request(url)

    # request.add_header('User-Agent', user_agent)
    fp = urlopen(request)

    if auth:
        # Put authentication info back into request URL if same host,
        # so that links found on the page will work
        s2, h2, path2, param2, query2, frag2 = urlparse(fp.url)
        if s2 == scheme and h2 == host:
            fp.url = urlunparse((s2, netloc, path2, param2, query2, frag2))

    return fp

 def load_posts(self, *, web_driver: WebDriver = None, **params) -> List[Post]:
     params.setdefault('owner_id', -self.group_id)
     if web_driver is None:
         raw_posts = self.get_all_objects('wall.get', **params)
     else:
         open_url('https://vk.com', web_driver)
         login = web_driver.find_element_by_xpath('//*[@id="index_email"]')
         login.clear()
         login.send_keys(self.user_login)
         password = web_driver.find_element_by_xpath('//*[@id="index_pass"]')
         password.clear()
         password.send_keys(self.user_password)
         web_driver.find_element_by_xpath('//*[@id="index_login_button"]').click()
         url_parts = list(urlparse('https://vk.com/dev/wall.get'))
         count = 100
         query = {'params[owner_id]': params['owner_id'],
                  'params[count]': count,
                  'params[offset]': params.get('offset', 0),
                  'params[filter]': params.get('filter', 'owner'),
                  'params[fields]': params.get('fields', ''),
                  'params[v]': self.api_version}
         url_parts[4] = urlencode(query)
         url = urlunparse(url_parts)
         response = parse_from_vk_dev(url, web_driver)['response']
         total_count = response['count']
         raw_posts = response['items']
         while len(raw_posts) < total_count:
             query['params[offset]'] += count
             url_parts[4] = urlencode(query)
             url = urlunparse(url_parts)
             response = parse_from_vk_dev(url, web_driver)['response']
             raw_posts += response['items']
     return [Post.from_raw(raw_post) for raw_post in raw_posts]

def split(url):
    """ Splits the path between root (scheme + FQDN) and rest of the URL

    Example::

        >>> split('http://example.com/foo')
        ('http://example.com', '/foo')
        >>> split('/foo/bar')
        ('', '/foo/bar')
        >>> split('https://user:[email protected]/foo/bar')
        ('https://user:[email protected]', '/foo/bar')
        >>> split('http://localhost/?foo=bar')
        ('http://localhost', '/?foo=bar')
        >>> split('http://localhost/foo?bar=baz')
        ('http://localhost', '/foo?bar=baz')
        >>> split('http://localhost?foo=bar')
        ('http://localhost', '/?foo=bar')

    :param url:     URL to strip the root from
    :returns:       path with root stripped
    """
    parsed = urlparse.urlparse(url)
    base = urlparse.urlunparse(mask(parsed, ROOT_MASK))
    stripped = urlparse.urlunparse(mask(parsed, TAIL_MASK))
    if stripped[0] != '/':
        stripped = '/' + stripped
    return base, stripped

    def __getattr__(self, name):
        if name == "urlWithoutVariables":
            return urlunparse((self.schema, self.__host, self.__path, '', '', ''))
        elif name == "pathWithVariables":
            return urlunparse(('', '', self.__path, '', self.__variablesGET.urlEncoded(), ''))
        elif name == "completeUrl":
            return urlunparse((self.schema, self.__host, self.__path, self.__params, self.__variablesGET.urlEncoded(), ''))
        elif name == "finalUrl":
            if self.__finalurl:
                return self.__finalurl
            return self.completeUrl
        elif name == "urlWithoutPath":
            return "%s://%s" % (self.schema, self._headers["Host"])
        elif name == "path":
            return self.__path
        elif name == "postdata":
            if self._non_parsed_post is not None:
                return self._non_parsed_post

            if self.ContentType == "application/x-www-form-urlencoded":
                return self.__variablesPOST.urlEncoded()
            elif self.ContentType == "multipart/form-data":
                return self.__variablesPOST.multipartEncoded()
            elif self.ContentType == 'application/json':
                return self.__variablesPOST.json_encoded()
            else:
                return self.__variablesPOST.urlEncoded()
        else:
            raise AttributeError

def request(url, method_name, debug=False, session_kwargs=None, **kwargs):
    session_kwargs = session_kwargs or {}
    parsed = urlparse(url)
    host = urlunparse(parsed[:2] + ('', '', '', ''))
    session = HttpSession(host, **session_kwargs)
    session.debug = debug

    path = urlunparse(('', '') + parsed[2:])
    return getattr(session, method_name)(path, **kwargs)

    def _request(self, path, method='GET', **params):
        """
            Make an API request
        """
        base = "{0}/{1}".format(self.baseurl, path)
        # normalise the method argument
        method = method.upper().strip()

        # parse the existing url
        urlparts = urlparse(base)
        qstr = parse_qs(urlparts.query)

        # add the token to the query string
        qstr['token'] = self.apikey
        if method not in APIClient.HTTP_REQUEST:
            qstr['_method'] = method
        else:
            try:
                del qstr['_method']
            except KeyError:
                pass

        if method in APIClient.ANY_GET_REQUEST:
            # if it's a get request then update the query string with
            # the params
            qstr.update(params)
            # all of the params go in the query string
            query_string = APIClient.urlencode(qstr)
            # reconstruct the url
            url = urlunparse((urlparts.scheme,
                              urlparts.netloc,
                              urlparts.path,
                              urlparts.params,
                              query_string,
                              "")) # empty fragment
            log.debug("Making GET request to {0}".format(url))
            resp, content = self.http.request(url, "GET", headers=self.USER_AGENT_HEADER)
        else:
            # all of the params go in the query string
            query_string = APIClient.urlencode(qstr)
            # reconstruct the url
            url = urlunparse((urlparts.scheme,
                              urlparts.netloc,
                              urlparts.path,
                              urlparts.params,
                              query_string,
                              "")) # empty fragment
            log.debug("Making POST request to {0}".format(url))
            resp, content = self.http.request(url, "POST", urlencode(params), headers=self.USER_AGENT_HEADER)

        status = int(resp['status'])

        if status != 200:
            raise APIError("An unknown error occurred")

        return resp, json.loads(content)

 def getPath(self, url):
     target = urlparse(url)
     
     targetPath = (target.path if target.path == "" or target.path.startswith("/")
                     else os.getcwd() + "/" + target.path) 
     fromUrlPart = urlunparse((self.url.scheme or "file", self.url.netloc, self.url.path,
                               "", "", ""))
     toUrlPart = urlunparse((target.scheme or "file", target.netloc, targetPath, 
                               "", "", ""))
     diffUrl = urljoin(fromUrlPart, toUrlPart)
     diffUrl = None if diffUrl == fromUrlPart else diffUrl
     return (diffUrl, target.fragment)

    def open_help(self, page, anchor):
        """
        Opens the specified help page. If <page> cannot be found, an error
        page will be opened. <anchor> will be attached to the url to enable
        the user to navigate inside a single html-page. The anchor will only
        be added, if one of the preferred browser is installed on the system
        (in order to prevent errors or strange behaviour).

        :param page: The page to open. The page is a html-file inside the
        folder 'help/<lang-code>/'.
        :param anchor: The anchor to attach to the url. Will be ignored if
        none of the prefered browsers is installed
        :return: The opened url as string
        """

        # First we create the path to the requested file. The file always is in
        # 'help/<lang>/'. If the file does exist, we append its name to the
        # path, if it does not exist, we append 'error.html', so the error-page
        # in the requested language will be displayed.
        requested = realpath(join(self.folder, str(page)))
        if not _does_file_exist_(requested):
            file_name = join(self.folder, "error.html")
            anchor = ''  # the error page does not need anchors
        else:
            file_name = requested

        # Here we try to get a handle for one of the prefered web browsers.
        # If we can get one, we take the handle and stop searching for another
        # handle. If we cannot get a handle (which means the browser is not
        # installed, we print an error message and take the next browser.
        web = None
        for browser in PREFERRED_BROWSERS:
            try:
                web = get(browser)
                break
            except Error:
                print("Was not able to open browser <" + str(browser) +
                      ">. Trying another...")

        # Here we open the url. The variable <web> will be <None> if we did not
        # find a handle for a prefered browser. So we open the url without
        # attaching the anchor (by requesting an automatic handle). Otherwise we
        # can append the url, because we can use one of the prefered browsers.
        if web:
            url = urlunparse(['file', '', realpath(file_name), '', '',
                              str(anchor)])
            web.open(url)
            return str(url)
        else:
            url = urlunparse(['file', '', realpath(file_name), '', '', ''])
            open(url)
            return str(url)

 def output(self, key, obj):
     if self.data_func:
         data = self.data_func(obj)
         o = urlparse(
             url_for(self.endpoint, _external=self.absolute, **data)
         )
         if self.absolute:
             scheme = self.scheme if self.scheme is not None else o.scheme
             return urlunparse(
                 (scheme, o.netloc, o.path, "", "", ""))
         return urlunparse(("", "", o.path, "", "", ""))
     else:
         return super(AbsoluteUrl, self).output(key, obj)

    def _generate_url(self, user_ids):
        user_ids = [str(user_id) for user_id in user_ids]
        params = {'user_ids': ','.join(user_ids), 'v': '5.45',
                  'access_token': VK_ACCESS_TOKEN}

        url_parts = list(urlparser.urlparse(self.VK_PHOTOS_API_URL))
        query = dict(urlparser.parse_qsl(url_parts[4]))
        query.update(params)

        url_parts[4] = urlencode(query)

        print(urlparser.urlunparse(url_parts))
        return urlparser.urlunparse(url_parts)

def campus_search():
    query = request.args.get('q')
    skip = int(request.args.get('skip', 0))
    limit = int(request.args.get('limit', 20))
    data = []

    if not query:
        campuses = Campus.objects().skip(skip).limit(limit)
    else:
        campuses = []

    for campus in campuses:
        campus_data = {
            'id': str(campus.id),
            'university': {
                'name': campus.univ_name,
                'type': campus.univ_type
            },
            'campus': {
                'name': campus.campus_name
            }
        }
        if campus.domain:
            campus_data['domain'] = campus.domain

        data.append(campus_data)

    result = {
        'data': data,
        'paging': {}
    }

    if skip > 0:
        url = list(parse.urlparse(request.url))
        query = dict(parse.parse_qs(url[4]))
        query['skip'] = skip - limit
        query['limit'] = limit
        if query['skip'] < 0:
            query['skip'] = 0
        url[4] = parse.urlencode(query)
        result['paging']['previous'] = parse.urlunparse(url)

    if len(data) >= limit:
        url = list(parse.urlparse(request.url))
        query = dict(parse.parse_qs(url[4]))
        query['skip'] = skip + len(data)
        query['limit'] = limit
        url[4] = parse.urlencode(query)
        result['paging']['next'] = parse.urlunparse(url)

    return jsonify(result)

    def url(self, val):
        """
        Parse url and set the url info into Page's object instances

        :type val : str
        """
        parsed_absolute_url = urlparse(self.to_absolute_url(val))

        #- Set: url, scheme, host, root_url
        self._scheme = parsed_absolute_url.scheme
        self._host = parsed_absolute_url.hostname
        self._root_url = urlunparse((parsed_absolute_url.scheme, parsed_absolute_url.netloc, "/", '', '', ''))
        normalized_path = parsed_absolute_url.path if parsed_absolute_url.path else "/"
        self._url = urlunparse((self._scheme, self.host, normalized_path,
                                parsed_absolute_url.params, parsed_absolute_url.query, parsed_absolute_url.fragment))

    def __repr__(self):
        if self.version:
            version = str(self.version)
        else:
            version = 'None'

        if self.url:
            url = str(urlunparse(self.url))
        elif self.unresolved_url:
            url = str(urlunparse(self.unresolved_url))
        else:
            url = 'Uninitialized'


        return self.__class__.__name__ + "(" + version + ', ' + url + ")"

def clean_urls(article_content, article_link):
    parsed_article_url = urlparse(article_link)
    parsed_content = BeautifulSoup(article_content, "html.parser")

    for img in parsed_content.find_all("img"):
        if "src" not in img.attrs:
            continue
        if is_secure_served() and "srcset" in img.attrs:
            # removing active content when serving over https
            del img.attrs["srcset"]
        to_rebuild, img_src = False, urlparse(img.attrs["src"])
        if not img_src.scheme or not img_src.netloc:
            to_rebuild = True
            # either scheme or netloc are missing from the src of the img
            scheme = img_src.scheme or parsed_article_url.scheme
            netloc = img_src.netloc or parsed_article_url.netloc
            img_src = ParseResult(
                scheme=scheme,
                netloc=netloc,
                path=img_src.path,
                query=img_src.query,
                params=img_src.params,
                fragment=img_src.fragment,
            )
        if to_rebuild:
            img.attrs["src"] = urlunparse(img_src)

    if is_secure_served():
        for iframe in parsed_content.find_all("iframe"):
            if "src" not in iframe.attrs:
                continue
            iframe_src = urlparse(iframe.attrs["src"])
            if iframe_src.scheme != "http":
                continue
            for domain in HTTPS_IFRAME_DOMAINS:
                if domain not in iframe_src.netloc:
                    continue
                iframe_src = ParseResult(
                    scheme="https",
                    netloc=iframe_src.netloc,
                    path=iframe_src.path,
                    query=iframe_src.query,
                    params=iframe_src.params,
                    fragment=iframe_src.fragment,
                )
                iframe.attrs["src"] = urlunparse(iframe_src)
                break
    return str(parsed_content)

def update_tracker(session_token, download_id, tracker):
    announce_url = tracker['announce']
    parts = list(urlparse(announce_url))
    parts[1] = NEW_TRACKER_HOST
    new_announce = urlunparse(parts)
    print(">  UPDATE tracker %s ==> %s" % (announce_url, new_announce))
    # add new tracker
    url = MAFREEBOX_API_URL + ("downloads/%d/trackers" % download_id)
    rep = requests.post(url, json={
        'announce': new_announce,
        'is_enabled': True
    }, headers={
        'X-Fbx-App-Auth': session_token
    })
    get_api_result(rep)

    # remove prev tracker
    url = MAFREEBOX_API_URL + ("downloads/%d/trackers/%s" % (download_id, quote(announce_url, safe='')))
    rep = requests.delete(url, headers={
        'X-Fbx-App-Auth': session_token
    })
    get_api_result(rep)

    # active new tracker
    url = MAFREEBOX_API_URL + ("downloads/%d/trackers/%s" % (download_id, quote(new_announce, safe='')))
    rep = requests.delete(url, json={
        'is_enabled': True
    }, headers={
        'X-Fbx-App-Auth': session_token
    })
    get_api_result(rep)

 def first_line(self):
     if not self._proxy and self.method != self.CONNECT:
         url = urlunparse(('', '', self.path or '/', self.params,
                           self.query, self.fragment))
     else:
         url = self.full_url
     return '%s %s %s' % (self.method, url, self.version)

def clean_service_url(url):
    """
    Return only the scheme, hostname and (optional) port components
    of the parameter URL.
    """
    parts = urlparse(url)
    return urlunparse((parts.scheme, parts.netloc, '', '', '', ''))

def relative_uri(source, target):
    """
    Make a relative URI from source to target.
    """
    su = urlparse.urlparse(source)
    tu = urlparse.urlparse(target)
    extra = list(tu[3:])
    relative = None
    if tu[0] == '' and tu[1] == '':
        if tu[2] == su[2]:
            relative = ''
        elif not tu[2].startswith('/'):
            relative = tu[2]
    elif su[0:2] != tu[0:2]:
        return target

    if relative is None:
        if tu[2] == su[2]:
            relative = ''
        else:
            relative = os.path.relpath(tu[2], os.path.dirname(su[2]))
    if relative == '.':
        relative = ''
    relative = urlparse.urlunparse(["", "", relative] + extra)
    return relative

 def output(self, key, obj):
     data = self._get_data(obj)
     data[self.key] = getattr(obj, self.attribute)
     endpoint = self.endpoint or request.endpoint
     # parse the url to remove the query string
     o = urlparse(url_for(endpoint, **data))
     return urlunparse(("", "", o.path, "", "", ""))

    def download(self, source, dest):
        """
        Download an archive file.

        :param str source: URL pointing to an archive file.
        :param str dest: Local path location to download archive file to.
        """
        # propogate all exceptions
        # URLError, OSError, etc
        proto, netloc, path, params, query, fragment = urlparse(source)
        if proto in ('http', 'https'):
            auth, barehost = splituser(netloc)
            if auth is not None:
                source = urlunparse((proto, barehost, path, params, query, fragment))
                username, password = splitpasswd(auth)
                passman = HTTPPasswordMgrWithDefaultRealm()
                # Realm is set to None in add_password to force the username and password
                # to be used whatever the realm
                passman.add_password(None, source, username, password)
                authhandler = HTTPBasicAuthHandler(passman)
                opener = build_opener(authhandler)
                install_opener(opener)
        response = urlopen(source)
        try:
            with open(dest, 'w') as dest_file:
                dest_file.write(response.read())
        except Exception as e:
            if os.path.isfile(dest):
                os.unlink(dest)
            raise e