def test_check_routes(app, io_loop, username, endpoints):
    proxy = app.proxy

    for endpoint in endpoints:
        r = api_request(app, endpoint, method='post')
        r.raise_for_status()

    test_user = orm.User.find(app.db, username)
    assert test_user is not None

    # check a valid route exists for user
    test_user = app.users[username]
    before = sorted(io_loop.run_sync(app.proxy.get_routes))
    assert unquote(test_user.proxy_path) in before

    # check if a route is removed when user deleted
    io_loop.run_sync(lambda: app.proxy.check_routes(app.users, app._service_map))
    io_loop.run_sync(lambda: proxy.delete_user(test_user))
    during = sorted(io_loop.run_sync(app.proxy.get_routes))
    assert unquote(test_user.proxy_path) not in during

    # check if a route exists for user
    io_loop.run_sync(lambda: app.proxy.check_routes(app.users, app._service_map))
    after = sorted(io_loop.run_sync(app.proxy.get_routes))
    assert unquote(test_user.proxy_path) in after

    # check that before and after state are the same
    assert before == after

def main():
    parser = argparse.ArgumentParser(description="Translate YAML `base_redirect` to .htaccess")
    parser.add_argument(
        "yaml_file",
        type=argparse.FileType("r"),
        default=sys.stdin,
        nargs="?",
        help="read from the YAML file (or STDIN)",
    )
    parser.add_argument(
        "htaccess_file",
        type=argparse.FileType("w"),
        default=sys.stdout,
        nargs="?",
        help="write to the .htaccess file (or STDOUT)",
    )
    args = parser.parse_args()

    # Load YAML document and look for 'entries' list.
    document = yaml.load(args.yaml_file)

    if not "idspace" in document or type(document["idspace"]) is not str:
        raise ValueError('YAML document must contain "idspace" string')
    idspace = document["idspace"]

    if not "base_url" in document or type(document["base_url"]) is not str:
        raise ValueError('YAML document must contain "base_url" string')

    if "base_redirect" in document and type(document["base_redirect"]) is str:
        base_url = unquote(document["base_url"])
        base_redirect = unquote(document["base_redirect"])
        args.htaccess_file.write(header_template % idspace)
        directive = 'RedirectMatch temp "(?i)^%s$" "%s"' % (base_url, base_redirect)
        args.htaccess_file.write(directive + "\n\n")

def smart_urlquote(url):
    "Quotes a URL if it isn't already quoted."
    def unquote_quote(segment):
        segment = unquote(segment)
        # Tilde is part of RFC3986 Unreserved Characters
        # http://tools.ietf.org/html/rfc3986#section-2.3
        # See also http://bugs.python.org/issue16285
        segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~'))
        return force_text(segment)

    # Handle IDN before quoting.
    try:
        scheme, netloc, path, query, fragment = urlsplit(url)
    except ValueError:
        # invalid IPv6 URL (normally square brackets in hostname part).
        return unquote_quote(url)

    try:
        netloc = netloc.encode('idna').decode('ascii')  # IDN -> ACE
    except UnicodeError:  # invalid domain part
        return unquote_quote(url)

    if query:
        # Separately unquoting key/value, so as to not mix querystring separators
        # included in query values. See #22267.
        query_parts = [(unquote(q[0]), unquote(q[1]))
                       for q in parse_qsl(query, keep_blank_values=True)]
        # urlencode will take care of quoting
        query = urlencode(query_parts)

    path = unquote_quote(path)
    fragment = unquote_quote(fragment)

    return urlunsplit((scheme, netloc, path, query, fragment))

 def normalizeUrl(self, url, base=None):
     if url and not (isHttpUrl(url) or os.path.isabs(url)):
         if base is not None and not isHttpUrl(base) and '%' in url:
             url = unquote(url)
         if base:
             if isHttpUrl(base):
                 scheme, sep, path = base.partition("://")
                 normedPath = scheme + sep + posixpath.normpath(os.path.dirname(path) + "/" + url)
             else:
                 if '%' in base:
                     base = unquote(base)
                 normedPath = os.path.normpath(os.path.join(os.path.dirname(base),url))
         else:
             normedPath = url
         if normedPath.startswith("file://"): normedPath = normedPath[7:]
         elif normedPath.startswith("file:\\"): normedPath = normedPath[6:]
         
         # no base, not normalized, must be relative to current working directory
         if base is None and not os.path.isabs(url): 
             normedPath = os.path.abspath(normedPath)
     else:
         normedPath = url
     
     if normedPath:
         if isHttpUrl(normedPath):
             scheme, sep, pathpart = normedPath.partition("://")
             pathpart = pathpart.replace('\\','/')
             endingSep = '/' if pathpart[-1] == '/' else ''  # normpath drops ending directory separator
             return scheme + "://" + posixpath.normpath(pathpart) + endingSep
         normedPath = os.path.normpath(normedPath)
         if normedPath.startswith(self.cacheDir):
             normedPath = self.cacheFilepathToUrl(normedPath)
     return normedPath

def service_url(url, defaults={}, extras_name='extras'):
    """
    environs handler for URLS, turning a url string into a dictionary
    of its component parts.
    """
    try:
        parsed = urlparse(url)
    except Exception:
        raise EnvError(
            'Service URLS look like: servtype://user:[email protected]:port/' +
            extras_name + '?param1=Foo&param2=Bar')

    conf = {
        'host': unquote(parsed.hostname) if parsed.hostname else None,
        'port': parsed.port,
        'user': unquote(parsed.username) if parsed.username else None,
        'password': unquote(parsed.password) if parsed.password else None,
        extras_name: unquote(parsed.path)[1:] if parsed.path else None,
    }

    # If we get multiple values for a given key, use the last.
    extra_params = {k: v[-1] for k, v in parse_qs(parsed.query).items()}
    conf.update(extra_params)

    missing_defaults = {k: v for k, v in defaults.items()
                        if k not in conf or conf[k] is None}
    conf.update(missing_defaults)

    return conf

def generate_aws_v4_signature(request):
    message = unquote(request.POST['to_sign'])
    dest = get_s3direct_destinations().get(unquote(request.POST['dest']))
    signing_date = datetime.strptime(request.POST['datetime'],
                                     '%Y%m%dT%H%M%SZ')

    auth = dest.get('auth')
    if auth and not auth(request.user):
        resp = json.dumps({'error': 'Permission denied.'})
        return HttpResponseForbidden(resp, content_type='application/json')

    region = getattr(settings, 'AWS_S3_REGION_NAME', None)
    if not region:
        resp = json.dumps({'error': 'S3 region config missing.'})
        return HttpResponseServerError(resp, content_type='application/json')

    aws_credentials = get_aws_credentials()
    if not aws_credentials.secret_key or not aws_credentials.access_key:
        resp = json.dumps({'error': 'AWS credentials config missing.'})
        return HttpResponseServerError(resp, content_type='application/json')

    signing_key = get_aws_v4_signing_key(aws_credentials.secret_key,
                                         signing_date, region, 's3')

    signature = get_aws_v4_signature(signing_key, message)
    resp = json.dumps({'s3ObjKey': signature})
    return HttpResponse(resp, content_type='application/json')

def parse_s3_url(url) -> Tuple[str, str, str, str]:
    """parses a s3 url and extract credentials and s3 object path.

    A S3 URL looks like s3://aws_key:[email protected]/objectname where
    credentials are optional. Since credentials might include characters
    such as `/`, `@` or `#`, they have to be urlquoted in the url.

    Args:
        url (str): the s3 url

    Returns:
        tuple: (access_key, secret, bucketname, objectname). If credentials
        are not specified, `access_key` and `secret` are set to None.
    """
    urlchunks = urlparse(url)
    scheme = urlchunks.scheme
    assert scheme in S3_SCHEMES, f'{scheme} unsupported, use one of {S3_SCHEMES}'
    assert not urlchunks.params, f's3 url should not have params, got {urlchunks.params}'
    assert not urlchunks.query, f's3 url should not have query, got {urlchunks.query}'
    assert not urlchunks.fragment, f's3 url should not have fragment, got {urlchunks.fragment}'
    # if either username or password is specified, we have credentials
    if urlchunks.username or urlchunks.password:
        # and they should both not be empty
        assert urlchunks.username, 's3 access key should not be empty'
        assert urlchunks.password, 's3 secret should not be empty'
        access_key = unquote(urlchunks.username)
        secret = unquote(urlchunks.password)
    else:
        access_key = secret = None
    objectname = urlchunks.path.lstrip('/')  # remove leading /, it's not part of the objectname
    assert objectname, f"s3 objectname can't be empty"
    return access_key, secret, urlchunks.hostname, objectname

    def translate_path(self, path):
        """
        Ignore the actual request path and just serve a specific folder.

        Mostly same as py3.6 source, replacing "path = os.getcwd()" with HERE so
        that the directory list or file transfers are relative to HERE rather
        than the current working directory.
        """
        # abandon query parameters
        path = path.split("?", 1)[0]
        path = path.split("#", 1)[0]
        # Don"t forget explicit trailing slash when normalizing. Issue17324
        trailing_slash = path.rstrip().endswith("/")
        try:
            path = unquote(path, errors="surrogatepass")
        except UnicodeDecodeError:
            path = unquote(path)
        except TypeError:  # py2 only accepts one param.
            path = unquote(path)
        path = posixpath.normpath(path)
        words = path.split("/")
        words = filter(None, words)
        path = HERE  # edited
        for word in words:
            if os.path.dirname(word) or word in (os.curdir, os.pardir):
                # Ignore components that are not a simple file/directory name
                continue
            path = os.path.join(path, word)
        if trailing_slash:
            path += "/"
        return path

    def parse(self, ticket):
        """Parses the passed ticket, returning a tuple containing the digest,
        user_id, valid_until, tokens, and user_data fields
        """
        if len(ticket) < self._min_ticket_size():
            raise TicketParseError(ticket, 'Invalid ticket length')

        digest_len = self._hash.digest_size * 2
        digest = ticket[:digest_len]

        try:
            time_len = 8
            time = int(ticket[digest_len:digest_len + time_len], 16)
        except:
            raise TicketParseError(ticket, 'Invalid time field')

        parts = ticket[digest_len + time_len:].split('!')
        if len(parts) != 3:
            raise TicketParseError(ticket, 'Missing parts')

        user_id = ulp.unquote(parts[0])
        tokens = ()
        if parts[1]:
            tokens = tuple((ulp.unquote(t) for t in parts[1].split(',')))

        user_data = ulp.unquote(parts[2])

        return TicketInfo(digest, user_id, tokens, user_data, time)

 def set(self, station, stream=None):
     station = unquote(station)
     if stream is not None:
         stream = unquote(stream)
     success = self.radio.set(station, stream)
     resp = 'Setting active stream to %s %s' % (station, stream)
     return json.dumps({'success': success, 'resp': resp}) + '\n'

    def normalize_parameters(params):
        """ Normalize parameters """
        params = params or {}
        normalized_parameters = OrderedDict()

        def get_value_like_as_php(val):
            """ Prepare value for quote """
            try:
                base = basestring
            except NameError:
                base = (str, bytes)

            if isinstance(val, base):
                return val
            elif isinstance(val, bool):
                return "1" if val else ""
            elif isinstance(val, int):
                return str(val)
            elif isinstance(val, float):
                return str(int(val)) if val % 1 == 0 else str(val)
            else:
                return ""

        for key, value in params.items():
            value = get_value_like_as_php(value)
            key = quote(unquote(str(key))).replace("%", "%25")
            value = quote(unquote(str(value))).replace("%", "%25")
            normalized_parameters[key] = value

        return normalized_parameters

def handle_client(client_reader, client_writer):

    req_line = yield from asyncio.wait_for(client_reader.readline(),
                                       timeout=10.0)
#   print('Req line "{}"'.format(req_line))
    while True:
        header = yield from asyncio.wait_for(client_reader.readline(),
                                       timeout=10.0)
        if header == b'\r\n':
            break
#       print('Header "{}"'.format(header))
        key, val = map(str.strip, header.rstrip().decode().lower().split(':', 1))

    method, path, version = req_line.decode().split(' ')
#   print('method = {!r}; path = {!r}; version = {!r}'.format(method, path, version))

    if path.startswith('/send_req'):
        path, args = path.split('?')
        args = loads(unquote(args))
        request_handler = RequestHandler()
        yield from request_handler.run(client_writer, args)
#           self.reader, self.writer, self.transport, self._request_handler, args)
    elif path.startswith('/get_login'):
        path, args = path.split('?')
        args = loads(unquote(args))
        request_handler = RequestHandler()
        yield from request_handler.get_login(client_writer, args)
    elif path.startswith('/dev'):
        path = path[4:]
        send_js_dev(client_writer, path)
    else:
        path = path[1:]
        send_js_prod(client_writer, path)

    def get_path(self, id_in_tree = None):

        id = ""
        if id_in_tree is None:

            if  (unquote(self.post["dir"])  == "0"):
                return True, ""
            id = unquote(self.post["dir"])
        else:
            id = id_in_tree

        dict_lbdoc = {"lb_ctrl_cookie": self.cookie ,
                      "lb_ctrl_op": "db_search",
                      "lb_ctrl_db": self.db_name,
                      "lb_ctrl_qry": "str_id_in_tree='" + id + "'",
                      'lb_ctrl_ctx': 'search_tree_file'}
        submit_operations_df_return = BrsLbdocCoreCs().submit_operations_df(None, dict_lbdoc, False)
        if submit_operations_df_return.operation_group[0].lb_occurrences_list:
            result = submit_operations_df_return.operation_group[0].lb_occurrences_list.results[0]

            return True,result.str_titulo

        else:
            a = ""
            lbdoc_return_objs = submit_operations_df_return.lbdoc_return_objs.lbdoc_return_objs[0]
            json_return = json.dumps(lbdoc_return_objs, default=lambda o: o.__dict__)
            return False,json_return

def baidu_get_song_data(sid):
    data = json.loads(get_html('http://music.baidu.com/data/music/fmlink?songIds=%s' % sid, faker = True))['data']

    if data['xcode'] != '':
    # inside china mainland
        return data['songList'][0]
    else:
    # outside china mainland
        html = get_html("http://music.baidu.com/song/%s" % sid)

        # baidu pan link
        sourceLink = r1(r'"link-src-info"><a href="([^"]+)"', html)
        if sourceLink != None:
            sourceLink = sourceLink.replace('&amp;', '&')
        sourceHtml = get_html(sourceLink) if sourceLink != None else None

        songLink =  r1(r'\\"dlink\\":\\"([^"]*)\\"', sourceHtml).replace('\\\\/', '/') if sourceHtml != None else r1(r'download_url="([^"]+)"', html)
        songName = parse.unquote(r1(r'songname=([^&]+)&', html))
        artistName = parse.unquote(r1(r'songartistname=([^&]+)&', html))
        albumName = parse.unquote(r1(r'songartistname=([^&]+)&', html))
        lrcLink = r1(r'data-lyricdata=\'{ "href":"([^"]+)"', html)

        return json.loads(json.dumps({'songLink'   : songLink,
                                      'songName'   : songName,
                                      'artistName' : artistName,
                                      'albumName'  : albumName,
                                      'lrcLink'    : lrcLink}, ensure_ascii=False))

 def hashed_name(self, name, content=None, filename=None):
     # `filename` is the name of file to hash if `content` isn't given.
     # `name` is the base name to construct the new hashed filename from.
     parsed_name = urlsplit(unquote(name))
     clean_name = parsed_name.path.strip()
     filename = (filename and urlsplit(unquote(filename)).path.strip()) or clean_name
     opened = content is None
     if opened:
         if not self.exists(filename):
             raise ValueError("The file '%s' could not be found with %r." % (filename, self))
         try:
             content = self.open(filename)
         except IOError:
             # Handle directory paths and fragments
             return name
     try:
         file_hash = self.file_hash(clean_name, content)
     finally:
         if opened:
             content.close()
     path, filename = os.path.split(clean_name)
     root, ext = os.path.splitext(filename)
     if file_hash is not None:
         file_hash = ".%s" % file_hash
     hashed_name = os.path.join(path, "%s%s%s" %
                                (root, file_hash, ext))
     unparsed_name = list(parsed_name)
     unparsed_name[2] = hashed_name
     # Special casing for a @font-face hack, like url(myfont.eot?#iefix")
     # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
     if '?#' in name and not unparsed_name[3]:
         unparsed_name[2] += '?'
     return urlunsplit(unparsed_name)

def _parse_qs(qs):
    pairs = (s2 for s1 in qs.split('&') for s2 in s1.split(';'))
    retval = odict()

    for name_value in pairs:
        if name_value is None or len(name_value) == 0:
            continue
        nv = name_value.split('=', 1)

        if len(nv) != 2:
            # Handle case of a control-name with no equal sign
            nv.append(None)

        name = unquote(nv[0].replace('+', ' '))

        value = None
        if nv[1] is not None:
            value = unquote(nv[1].replace('+', ' '))

        l = retval.get(name, None)
        if l is None:
            l = retval[name] = []
        l.append(value)

    return retval

    def searchForReleaseImages(self, artistName, releaseTitle, query=None):

        result = []
        query = parse.unquote(query)
        if query and self.regex.match(parse.unquote(query)):
            result.append(ImageSearchResult(0, 0, query))
            return result

        try:
            bingResult = self.bingSearchImages(query if query else artistName + " " + releaseTitle)
            if bingResult:
                for br in bingResult:
                    result.append(br)
        except:
            pass

        try:
            it = self.itunesSearchArtistReleaseImages(artistName, releaseTitle)
            if it:
                for i in it:
                    result.append(i)
        except:
            pass

        return result

def defang(url):
    try:
        unquote(url)
        u = urlparse()
    except:
        return "Invalid URL"

    # hxxps:\/\/www[dot]cwi[dot]nl
    defanged_url = f"{u.scheme.replace('t', 'x')}:\/\/{u.hostname.replace('.', '[dot]')}"

    if u.port:
        defanged_url += ':' + str(u.port)

    if u.path:
        defanged_url += u.path.replace('/', '\/')

    if u.params:
        defanged_url += u.params.replace('/', '\/').replace('.', '[dot]')

    if u.query:
        defanged_url += '?' + u.query.replace('/', '\/').replace('.', '[dot]')

    if u.fragment:
        defanged_url += '#' + u.fragment

    return defanged_url

def decode(encoded_str):
    """Decode an encrypted HTTP basic authentication string. Returns a tuple of
    the form (username, password), and raises a DecodeError exception if
    nothing could be decoded.
    """
    split = encoded_str.strip().encode("latin1").split(None)

    # If split is only one element, try to decode the username and password
    # directly.
    if len(split) == 1:
        try:
            username, password = b64decode(split[0]).split(b':', 1)
        except:
            raise DecodeError

    # If there are only two elements, check the first and ensure it says
    # 'basic' so that we know we're about to decode the right thing. If not,
    # bail out.
    elif len(split) == 2:
        if split[0].strip().lower() == b'basic':
            try:
                username, password = b64decode(split[1]).split(b':', 1)
            except:
                raise DecodeError
        else:
            raise DecodeError

    # If there are more than 2 elements, something crazy must be happening.
    # Bail.
    else:
        raise DecodeError

    return (unquote(username.decode('latin1')),
            unquote(password.decode('latin1')))

def find_princs_digest(param, request):
    sess = DBSession()

    try:
        user = sess.query(User).filter(User.state == UserState.active,
                                       User.enabled.is_(True),
                                       User.login == param['username']).one()
    except NoResultFound:
        return None
    if not user.password_ha1:
        return None
    req_path = unquote(request.path.lower())
    uri_path = unquote(param['uri'].lower())
    if req_path != uri_path:
        return None
    ha2 = hashlib.md5(('%s:%s' % (request.method,
                                  param['uri'])).encode()).hexdigest()
    data = '%s:%s:%s:%s:%s' % (param['nonce'], param['nc'],
                               param['cnonce'], 'auth', ha2)
    resp = hashlib.md5(('%s:%s' % (user.password_ha1,
                                   data)).encode()).hexdigest()
    if hmac.compare_digest(resp, param['response']):
        groups = ['g:%s' % (user.group.name,)]
        for sgr in user.secondary_groups:
            if sgr == user.group:
                continue
            groups.append('g:%s' % (sgr.name,))
        return groups
    return None