def smokeTest(self):
     falseNegatives = 0.
     falsePositives = 0.
     pCount = 0.
     nCount = 0.
     for article in os.listdir(self.parentURL + "positive"):
         pCount += 1
         s = join(self.parentURL+"positive",article)
         result, resultString = self.evaluate(urllib.pathname2url(s))
         if result:
             continue
         else:
             falseNegatives += 1
     pError = falseNegatives / pCount
     print "Rate of False Negatives = ", pError*100.0
     for article in os.listdir(self.parentURL + "negative"):
         nCount += 1
         s = join(self.parentURL+"negative",article)
         result, resultString = self.evaluate(urllib.pathname2url(s))
         if not result:
             continue
         else:
             falsePositives += 1
     nError = falsePositives / nCount
     print "Rate of False Positives = ", nError*100.0
     accuracy = (((nCount - falsePositives) + (pCount - falseNegatives)) / (nCount + pCount))*100.0
     print "Total Accuracy = ", accuracy
     print falseNegatives
     print pCount
     print falsePositives
     print ncount

    def _set_url(self, wsdl):
        """ 
        Set the path of file-based wsdls for processing.If not file-based,
        return a fully qualified url to the WSDL
        """
        if self.fromurl == True:
            if wsdl.endswith('wsdl'):
                wsdl.replace('.wsdl','')
            qstring = '?WSDL=%s' % wsdl
            return 'https://%s:%s%s' % (self.hostname, self.port,
                                        ICONTROL_URI + qstring)

        else:

            if wsdl.endswith('wsdl'):
                pass
            else:
                wsdl = wsdl + '.wsdl'
        
            # Check for windows and use goofy paths. Otherwise assume *nix
            if platform.system().lower() == 'windows':
                url = 'file:' + pathname2url(self.directory +'\\' + wsdl)
            else:
                url = 'file:' + pathname2url(self.directory + '/' + wsdl)
        return url

def get_user_id(username, location):
    """Gets the user id for a particular user.

    Args:
        username: URL encoded username for the summoner.
        location: Riot abbreviation for the region.
    """

    try:
        LOGGING.push(
            "*'" + username + "'* from @'" + location +
            "'@ is requesting their user ID."
        )

        # TODO(Save the ID lookup in the database.)
        session = RiotSession(API_KEY, location)

        response = session.get_ids([urllib.pathname2url(username)])
        return response[urllib.pathname2url(username)]['id']

    # TODO(Fix this to catch both 429 and 400 errors w/ Riot Exception.)
    except ValueError:
        LOGGING.push(
            "Tried to get *'" + username +
            "'* id. Response did not have user id."
        )
        abort(404, {'message': "User ID was not found."})

 def test_quoting(self):
     # Test automatic quoting and unquoting works for pathnam2url() and
     # url2pathname() respectively
     given = os.path.join("needs", "quot=ing", "here")
     expect = "needs/%s/here" % urllib.quote("quot=ing")
     result = urllib.pathname2url(given)
     self.assertEqual(expect, result,
                      "pathname2url() failed; %s != %s" %
                      (expect, result))
     expect = given
     result = urllib.url2pathname(result)
     self.assertEqual(expect, result,
                      "url2pathname() failed; %s != %s" %
                      (expect, result))
     given = os.path.join("make sure", "using_quote")
     expect = "%s/using_quote" % urllib.quote("make sure")
     result = urllib.pathname2url(given)
     self.assertEqual(expect, result,
                      "pathname2url() failed; %s != %s" %
                      (expect, result))
     given = "make+sure/using_unquote"
     expect = os.path.join("make+sure", "using_unquote")
     result = urllib.url2pathname(given)
     self.assertEqual(expect, result,
                      "url2pathname() failed; %s != %s" %
                      (expect, result))

 def generate_filename(self, title, content, date):
     if title:
         title = title.replace(' ', '-')
         return urllib.pathname2url(title)
     else:
         hash = hashlib.sha256(content + date).digest()
         return urllib.pathname2url(hash)

 def get_resource(self, url, hint=None):
   # Try to open as a local path
   try:
     handle = urllib.urlopen(urllib.pathname2url(url))
     return handle
   except IOError as e:
     pass
   # Try to open as an absolute path by combining 'hint' and 'url'
   if hint is not None:
     try:
       path = os.path.join(hint, url)
       handle = urllib.urlopen(urllib.pathname2url(path))
       return handle
     except IOError as e:
       pass
   # Case where input URL is not a local path
   try:
     handle = self.opener.open(url)
     return handle
   except IOError as e:
     pass
   if not hasattr(e, 'code') or e.code != 401:
     raise EnvironmentError(str(e.errno) + ": " + str(e.strerror))
   # Case where login / password are unknown
   else:
     username = str(raw_input("Username for " + url + ": "))
     password = getpass.getpass()
     self.manager.add_password(None, url, username, password)
     try:
       handle = self.opener.open(url)
       return handle
     except IOError as e:
       print(str(e.errno) + ": " + str(e.strerror))

def getRoUri(roref):
    uri = roref
    if urlparse.urlsplit(uri).scheme == "":
        base = "file://"+urllib.pathname2url(os.path.abspath(os.getcwd()))+"/"
        uri  = urlparse.urljoin(base, urllib.pathname2url(roref))
    if not uri.endswith("/"): uri += "/" 
    return rdflib.URIRef(uri)

def check(tool, mainFD):
  checkFD=codecs.open(pj(args.reportOutDir, tool, "check.txt"), "w", encoding="utf-8")
  if not args.disableMakeCheck:
    # make check
    print("RUNNING make check\n", file=checkFD); checkFD.flush()
    if simplesandbox.call(["make", "-j", str(args.j), "check"], envvar=["PKG_CONFIG_PATH", "LD_LIBRARY_PATH"], shareddir=["."],
                          stderr=subprocess.STDOUT, stdout=checkFD)==0:
      result="done"
    else:
      result="failed"
  else:
    print("make check disabled", file=checkFD); checkFD.flush()
    result="done"

  foundTestSuiteLog=False
  testSuiteLogFD=codecs.open(pj(args.reportOutDir, tool, "test-suite.log.txt"), "w", encoding="utf-8")
  for rootDir,_,files in os.walk('.'): # append all test-suite.log files
    if "test-suite.log" in files:
      testSuiteLogFD.write('\n\n')
      testSuiteLogFD.write(open(pj(rootDir, "test-suite.log")).read())
      foundTestSuiteLog=True
  testSuiteLogFD.close()
  if not args.disableMakeCheck:
    print('<td class="%s"><span class="glyphicon glyphicon-%s"></span>&nbsp;'%("success" if result=="done" else "danger",
      "ok-sign alert-success" if result=="done" else "exclamation-sign alert-danger"), file=mainFD)
    print('  <a href="'+myurllib.pathname2url(pj(tool, "check.txt"))+'">'+result+'</a>', file=mainFD)
    if foundTestSuiteLog:
      print('  <a href="'+myurllib.pathname2url(pj(tool, "test-suite.log.txt"))+'">test-suite.log</a>', file=mainFD)
    print('</td>', file=mainFD)
  checkFD.close()
  mainFD.flush()

  if result!="done":
    return 1
  return 0

def create_input_source(source=None, publicID=None,
                        location=None, file=None, data=None, format=None):
    """
    Return an appropriate InputSource instance for the given
    parameters.
    """

    # TODO: test that exactly one of source, location, file, and data
    # is not None.

    input_source = None

    if source is not None:
        if isinstance(source, InputSource):
            input_source = source
        else:
            if isinstance(source, basestring):
                location = source
            elif hasattr(source, "read") and not isinstance(source, Namespace):
                f = source
                input_source = InputSource()
                input_source.setByteStream(f)
                if hasattr(f, "name"):
                    input_source.setSystemId(f.name)
            else:
                raise Exception("Unexpected type '%s' for source '%s'" %
                                (type(source), source))

    absolute_location = None  # Further to fix for issue 130

    if location is not None:
        # Fix for Windows problem https://github.com/RDFLib/rdflib/issues/145
        if os.path.exists(location):
            location = pathname2url(location)
        base = urljoin("file:", "%s/" % pathname2url(os.getcwd()))
        absolute_location = URIRef(location, base=base).defrag()
        if absolute_location.startswith("file:///"):
            filename = url2pathname(absolute_location.replace("file:///", "/"))
            file = open(filename, "rb")
        else:
            input_source = URLInputSource(absolute_location, format)
        # publicID = publicID or absolute_location # More to fix for issue 130

    if file is not None:
        input_source = FileInputSource(file)

    if data is not None:
        if isinstance(data, unicode):
            data = data.encode('utf-8')
        input_source = StringInputSource(data)

    if input_source is None:
        raise Exception("could not create InputSource")
    else:
        if publicID is not None:  # Further to fix for issue 130
            input_source.setPublicId(publicID)
        # Further to fix for issue 130
        elif input_source.getPublicId() is None:
            input_source.setPublicId(absolute_location or "")
        return input_source

                def handle_result_pixbuf(pixbuf, engine_uri, tooltip_image, tooltip_text, should_save):
                    if self.ticket.release(entry, ticket):
                        if should_save:
                            if pixbuf.get_has_alpha():
                                pixbuf.savev(
                                    art_location_png,
                                    ART_CACHE_FORMAT_PNG,
                                    ART_CACHE_SETTINGS_NAMES_PNG,
                                    ART_CACHE_SETTINGS_VALUES_PNG,
                                )
                                uri = "file://" + pathname2url(art_location_png)
                            else:
                                pixbuf.savev(
                                    art_location_jpg,
                                    ART_CACHE_FORMAT_JPG,
                                    ART_CACHE_SETTINGS_NAMES_JPG,
                                    ART_CACHE_SETTINGS_VALUES_JPG,
                                )
                                uri = "file://" + pathname2url(art_location_jpg)

                            self.write_meta_file(art_location_meta, tooltip_image, tooltip_text)
                        else:
                            uri = engine_uri

                        print "found image for %s" % (entry.get_string(RB.RhythmDBPropType.LOCATION))
                        callback(entry, pixbuf, uri, tooltip_image, tooltip_text)
                        for m in self.same_search.pop(entry, []):
                            print "and for same search %s" % (m.get_string(RB.RhythmDBPropType.LOCATION))
                            callback(m, pixbuf, uri, tooltip_image, tooltip_text)

                    self.write_blist(blist_location, blist)
                    self.same_search.pop(entry, None)

    def update_submission(submission, status, job_id):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        if status == 'running':
            _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            state = {}
            if len(submission.execution_key) > 0:
                logger.debug("update_submission_task loading state: %s", submission.execution_key)
                state = json.loads(submission.execution_key)
            if 'score' in state:
                logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
                submission.output_file.name = pathname2url(submission_output_filename(submission))
                submission.save()
                logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = open(ozip.extract('scores.txt'), 'r').read()
                logger.debug("Processing scores... (submission_id=%s)", submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        try:
                            scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
                                                                      key=label.strip())
                            SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)", submission.id)
                _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
                result = Job.FINISHED
            else:
                logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
                url_name = pathname2url(submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
                except Exception:
                    logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
        _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)

def setup_psd_pregenerated(workflow, tags=[]):
    '''
    Setup CBC workflow to use pregenerated psd files.
    The file given in cp.get('workflow','pregenerated-psd-file-(ifo)') will 
    be used as the --psd-file argument to geom_nonspinbank, geom_aligned_bank
    and pycbc_plot_psd_file.

    Parameters
    ----------
    workflow: pycbc.workflow.core.Workflow
        An instanced class that manages the constructed workflow.
    tags : list of strings
        If given these tags are used to uniquely name and identify output files
        that would be produced in multiple calls to this function.

    Returns
    --------
    psd_files : pycbc.workflow.core.FileList
        The FileList holding the gating files
    '''
    psd_files = FileList([])

    cp = workflow.cp
    global_seg = workflow.analysis_time
    user_tag = "PREGEN_PSD"

    # Check for one psd for all ifos
    try:
        pre_gen_file = cp.get_opt_tags('workflow-psd',
                        'psd-pregenerated-file', tags)
        pre_gen_file = resolve_url(pre_gen_file)
        file_url = urlparse.urljoin('file:',
                                     urllib.pathname2url(pre_gen_file))
        curr_file = File(workflow.ifos, user_tag, global_seg, file_url,
                                                    tags=tags)
        curr_file.PFN(file_url, site='local')
        psd_files.append(curr_file)
    except ConfigParser.Error:
        # Check for one psd per ifo
        for ifo in workflow.ifos:
            try:
                pre_gen_file = cp.get_opt_tags('workflow-psd',
                                'psd-pregenerated-file-%s' % ifo.lower(),
                                tags)
                pre_gen_file = resolve_url(pre_gen_file)
                file_url = urlparse.urljoin('file:',
                                             urllib.pathname2url(pre_gen_file))
                curr_file = File(ifo, user_tag, global_seg, file_url,
                                                            tags=tags)
                curr_file.PFN(file_url, site='local')
                psd_files.append(curr_file)

            except ConfigParser.Error:
                # It's unlikely, but not impossible, that only some ifos
                # will have pregenerated PSDs
                logging.warn("No psd file specified for IFO %s." % (ifo,))
                pass
            
    return psd_files

 def _get_base_url(self, scheme, host, port, file_path):
     if netutils.is_valid_ipv6(host):
         base_url = "%s://[%s]:%s/folder/%s" % (scheme, host, port,
                                             urllib.pathname2url(file_path))
     else:
         base_url = "%s://%s:%s/folder/%s" % (scheme, host, port,
                                           urllib.pathname2url(file_path))
     return base_url

 def _get_base_url(self, scheme, host, port, file_path):
     if netutils.is_valid_ipv6(host):
         base_url = "{0!s}://[{1!s}]:{2!s}/folder/{3!s}".format(scheme, host, port,
                                             urllib.pathname2url(file_path))
     else:
         base_url = "{0!s}://{1!s}:{2!s}/folder/{3!s}".format(scheme, host, port,
                                           urllib.pathname2url(file_path))
     return base_url

def test_post_filename():

    content = 'some stuff in here'
    title = 'this is awesome'
    post = Post(content, prefix=prefix, author=author)
    expected = urllib.pathname2url(hashlib.sha256(content + post.meta['date']).digest())
    assert post.filename == expected, 'filename is %s expected %s' % (post.filename, expected)
    post = Post(content, prefix=prefix, title=title, author=author)
    assert post.filename == urllib.pathname2url(title.replace(' ', '-'))

    def parse_grabbed(self, url, item, datestr=u'2014-07-18T11:20:24+00:00'):
        """
        Parse grabbed item, extract title content, tags
        THIS IS THE METHOD YOU HAVE TO MODIFY FIRST TO GET THIS THING WORKING
        """
        raw_data = item['raw_content']
        parsd = PyQuery(raw_data)
        content_el = parsd('div.entry-content')
        if not content_el:
            content_el = parsd('.post-content')
        content = content_el.html()
        title = parsd('h1').html()
        tags = []
        for raw_tag in parsd('ul.tag-list>li>a'):
            tag = {'title':raw_tag.text,
                   'slug':urllib.pathname2url(
                       raw_tag.attrib['href'].split('/')[-1].encode('utf8')
                    )
                }
            tags.append(tag)
        raw_posted_date = parsd('header .entry-meta time.entry-date')
        if raw_posted_date:
            raw_posted_date_text = raw_posted_date[0].attrib['datetime']
        else:
            print "Failed to parse date!"
            raw_posted_date_text=datestr
        print "Setting post date: {}".format(raw_posted_date_text)
        posted_date = datetime.datetime.strptime(raw_posted_date_text[:-6],"%Y-%m-%dT%H:%M:%S")
        raw_category = None
        for potential_category in parsd('a'):
            if potential_category.attrib.get('rel'):
                if 'tag' in potential_category.attrib.get('rel'):
                    raw_category = potential_category
                    break
        if raw_category:
            category = {'title':raw_category.text,
                        'slug':urllib.pathname2url(
                            raw_category.attrib['href'].split('/')[-1].encode('utf8')
                        )}
        else:
            category = None
        author_raw = parsd('header vcard>a')
        author = author_raw[0].text if author_raw else None

        Posts.update({'url':url},{'$set':{
            'slug':url.split('/')[-1],
            'content':content,
            'title':title,
            'tags':tags,
            'posted_date':posted_date,
            'category':category,
            'author':author,
            'parsed':True
        }})
        self.parsedurls += 1
        time.sleep(1)
        return Posts.find_one({'url':url})

def generate_capture(exepath, html_path, img_path):
    """  HTML画面キャプチャを取得し、png形式で出力。

    * exepath: コマンド
    * html_path: HTMLファイルパス
    * img_path: 結果出力パス
    """

    timeout = 10

    try:
        if IS_WINDOWS:
            page_uri = 'file:%s' % urllib.pathname2url(os.path.normpath(html_path))
            exepath += '.exe'
            cmd = [exepath, '--url=%s' % page_uri, '--out=%s' % img_path, '--max-wait=%s' % 30000]
            proc = subprocess.Popen(cmd, creationflags=CREATE_NO_WINDOW, bufsize=-1)
        else:
            page_uri = 'file://%s' % urllib.pathname2url(os.path.normpath(html_path))
            exepath += '64' if '64' in platform.machine() else ''
            #if os.getenv('DISPLAY'):
            cmd = [exepath, '--url=%s' % page_uri, '--out=%s' % img_path, '--max-wait=%s' % 30000]
            #else:
            #    cmd = ['xvfb-run', '-s', '-screen 0, 1024x768x24', exepath, '--url=%s' % page_uri, '--out=%s' % img_path]
            proc = subprocess.Popen(cmd, bufsize=-1)
        st = time.time()
        while time.time() - st <= timeout:
            if proc.poll() is not None:
                try:
                    triming(img_path)
                except:
                    raise
                finally:
                    return True
            else:
                time.sleep(0.1)

        logger.debug('dead process %s' % proc.pid)
        try:
            proc.terminate()
        except:
            logger.debug(traceback.format_exc())

        time.sleep(1)
        if proc.poll() is not None:
            try:
                proc.kill()
            except:
                logger.debug(traceback.format_exc())

        return False
    except:
        logger.debug(traceback.format_exc())
        #logger.debug('exepath = %s' % (exepath))
        #logger.debug('Error occured while tring to generate capture for %s -> %s' % (html_path, img_path))
        raise

 def setUp(self):
     fd1, self.filename1 = mkstemp(suffix=self.serpar.EXTENSION,
                                   prefix="advene2_utest_serpar_")
     fd2 , self.filename2 = mkstemp(suffix=self.serpar.EXTENSION,
                                    prefix="advene2_utest_serpar_")
     fdopen(fd1).close()
     fdopen(fd2).close()
     self.url = "file:" + pathname2url(self.filename2)
     self.p1 = self.pkgcls("file:" + pathname2url(self.filename1),
                           create=True)
     self.p2 = None

 def getUrlPath(self, source):
     path = self.__makepath(source)
     # logger.debug(path)
     try:
         upath = urllib.pathname2url(str(path))
     except UnicodeEncodeError:
         upath = urllib.pathname2url(path.encode('utf-8'))
     if upath[:3]=='///':
         return 'file:' + upath
     else:
         return 'file://' + upath

 def getSearchItemURL(self, searchItem, page):
     url = "http://www.zhaoonline.com/search/"
     url += urllib.pathname2url(searchItem.name)
     url += "-8-3-trade-"
     url += urllib.pathname2url(str(categoryDic[searchItem.category]))
     url += "-"
     url += urllib.pathname2url(str(qualityDic[searchItem.quality]))
     url += "-00-N-0-N-1-"
     url += str(page)
     url += ".htm"
     return  url