def import_products(provider_name, import_input):
    if provider_name in ["bibtex", "product_id_strings"]:
        logger.debug(u"in import_products with provider_name {provider_name}".format(
            provider_name=provider_name))
    else:
        logger.debug(u"in import_products with provider_name {provider_name}: {import_input}".format(
            provider_name=provider_name, import_input=import_input))

    aliases = []

    # pull in standard items, if we were passed any of these
    if provider_name=="product_id_strings":
        aliases = get_aliases_from_product_id_strings(import_input)
    elif provider_name=="bibtex":
        provider_module = ProviderFactory.get_provider("bibtex")
        aliases = provider_module.member_items(import_input)
    else:
        try:
            provider_module = ProviderFactory.get_provider(provider_name)
            aliases = provider_module.member_items(import_input)
        except ImportError:
            logger.debug(u"in import_products, got ImportError")
            pass

    # logger.debug(u"returning from import_products with aliases {aliases}".format(
    #     aliases=aliases))

    return(aliases)

    def setUp(self):
        self.config = None #placeholder
        self.TEST_PROVIDER_CONFIG = [
            ("wikipedia", {})
        ]
        # hacky way to delete the "ti" db, then make it fresh again for each test.
        temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB"))
        temp_dao.delete_db(os.getenv("CLOUDANT_DB"))
        self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB"))

        # do the same thing for the redis db, set up the test redis database.  We're using DB Number 8
        self.r = tiredis.from_url("redis://localhost:6379", db=8)
        self.r.flushdb()

        provider_queues = {}
        providers = ProviderFactory.get_providers(self.TEST_PROVIDER_CONFIG)
        for provider in providers:
            provider_queues[provider.provider_name] = backend.PythonQueue(provider.provider_name+"_queue")

        self.b = backend.Backend(
            backend.RedisQueue("alias-unittest", self.r), 
            provider_queues, 
            [backend.PythonQueue("couch_queue")], 
            self.r)

        self.fake_item = {
            "_id": "1",
            "type": "item",
            "num_providers_still_updating":1,
            "aliases":{"pmid":["111"]},
            "biblio": {},
            "metrics": {}
        }
        self.fake_aliases_dict = {"pmid":["222"]}
        self.tiid = "abcd"

    def setUp(self):
        self.config = None #placeholder
        self.TEST_PROVIDER_CONFIG = [
            ("wikipedia", {})
        ]
        self.d = None

        # do the same thing for the redis db, set up the test redis database.  We're using DB Number 8
        self.r = tiredis.from_url("redis://localhost:6379", db=8)
        self.r.flushdb()

        provider_queues = {}
        providers = ProviderFactory.get_providers(self.TEST_PROVIDER_CONFIG)
        for provider in providers:
            provider_queues[provider.provider_name] = backend.PythonQueue(provider.provider_name+"_queue")

        self.b = backend.Backend(
            backend.RedisQueue("alias-unittest", self.r), 
            provider_queues, 
            [backend.PythonQueue("couch_queue")], 
            self.r)

        self.fake_item = {
            "_id": "1",
            "type": "item",
            "num_providers_still_updating":1,
            "aliases":{"pmid":["111"]},
            "biblio": {},
            "metrics": {},
            "last_modified": datetime.datetime(2013, 1, 1)
        }
        self.fake_aliases_dict = {"pmid":["222"]}
        self.tiid = "abcd"

        self.db = setup_postgres_for_unittests(db, app)

def create_item(namespace, nid, myredis, mydao):
    logger.debug("In create_item with alias" + str((namespace, nid)))
    item = make()
    namespace = clean_id(namespace)
    nid = clean_id(nid)
    item["aliases"][namespace] = [nid]
    item["aliases"] = canonical_aliases(item["aliases"])

    # set this so we know when it's still updating later on
    myredis.set_num_providers_left(
        item["_id"],
        ProviderFactory.num_providers_with_metrics(default_settings.PROVIDERS)
    )

    mydao.save(item)

    myredis.add_to_alias_queue(item["_id"], item["aliases"])

    logger.info("Created new item '{id}' with alias '{alias}'".format(
        id=item["_id"],
        alias=str((namespace, nid))
    ))

    mixpanel.track("Create:Item", {"Namespace":namespace})

    try:
        return item["_id"]
    except AttributeError:
        abort(500)

def importer_post(provider_name):
    """
    Gets aliases associated with a query from a given provider.
    """
    input_string = request.json["input"]

    if provider_name == "pmids":
        provider_name = "pubmed"
    elif provider_name == "dois":
        provider_name = "crossref"
    elif provider_name == "urls":
        provider_name = "webpage"
    try:
        provider = ProviderFactory.get_provider(provider_name)
    except ImportError:
        abort_custom(404, "an importer for provider '{provider_name}' is not found".format(
            provider_name=provider_name))

    try:
        aliases = provider.member_items(input_string)
    except ProviderItemNotFoundError:
        abort_custom(404, "item not found")
    except (ProviderTimeout, ProviderServerError):
        abort_custom(503, "timeout error, might be transient")
    except ProviderError:
        abort(500, "internal error from provider")

    tiids_aliases_map = item_module.create_tiids_from_aliases(aliases, myredis)
    logger.debug(u"in provider_importer_get with {tiids_aliases_map}".format(
        tiids_aliases_map=tiids_aliases_map))

    products_dict = format_into_products_dict(tiids_aliases_map)

    resp = make_response(json.dumps({"products": products_dict}, sort_keys=True, indent=4), 200)
    return resp

def sniffer(item_aliases, provider_config=default_settings.PROVIDERS):

    (genre, host) = item_module.decide_genre(item_aliases)

    all_metrics_providers = [provider.provider_name for provider in 
                    ProviderFactory.get_providers(provider_config, "metrics")]

    if (genre == "article") and (host != "arxiv"):
        run = [[("aliases", provider)] for provider in ["mendeley", "crossref", "pubmed", "altmetric_com"]]
        run += [[("biblio", provider) for provider in ["crossref", "pubmed", "mendeley", "webpage"]]]
        run += [[("metrics", provider) for provider in all_metrics_providers]]
    elif (host == "arxiv") or ("doi" in item_aliases):
        run = [[("aliases", provider)] for provider in [host, "altmetric_com"]]
        run += [[("biblio", provider) for provider in [host, "mendeley"]]]
        run += [[("metrics", provider) for provider in all_metrics_providers]]
    else:
        # relevant alias and biblio providers are always the same
        relevant_providers = [host]
        if relevant_providers == ["unknown"]:
            relevant_providers = ["webpage"]
        run = [[("aliases", provider)] for provider in relevant_providers]
        run += [[("biblio", provider) for provider in relevant_providers]]
        run += [[("metrics", provider) for provider in all_metrics_providers]]

    return(run)

def provider_run(aliases_dict, tiid, method_name, provider_name):

    provider = ProviderFactory.get_provider(provider_name)

    # logger.info(u"in provider_run for {provider}".format(
    #    provider=provider.provider_name))

    (success, estimated_wait_seconds) = rate.acquire(provider_name, block=False)
    # add up to random 2 seconds to spread it out
    estimated_wait_seconds += random.random() * 3
    if not success:
        logger.warning(u"RATE LIMIT HIT in provider_run for {provider} {method_name} {tiid}, retrying".format(
           provider=provider.provider_name, method_name=method_name, tiid=tiid))
        provider_run.retry(args=[aliases_dict, tiid, method_name, provider_name],
                countdown=estimated_wait_seconds, 
                max_retries=10)

    timeout_seconds = 30
    try:
        with timeout.Timeout(timeout_seconds):
            response = provider_method_wrapper(tiid, aliases_dict, provider, method_name)

    except timeout.Timeout:
        msg = u"TIMEOUT in provider_run for {provider} {method_name} {tiid} after {timeout_seconds} seconds".format(
           provider=provider.provider_name, method_name=method_name, tiid=tiid, timeout_seconds=timeout_seconds)
        # logger.warning(msg)  # message is written elsewhere
        raise ProviderTimeout(msg)

    return response

def provider_memberitems(provider_name):
    """
    Starts a memberitems update for a specified provider, using a supplied file.

    Returns a hash of the file's contents, which is needed to get memberitems'
    output. To get output, poll GET /provider/<provider_name>/memberitems/<hash>?method=async
    """
    # logger.debug("Query POSTed to {provider_name}/memberitems with request headers '{headers}'".format(
    #    provider_name=provider_name,
    #    headers=request.headers
    # ))

    file = request.files["file"]
    logger.debug("In provider_memberitems got file")
    logger.debug("filename = " + file.filename)
    query = file.read().decode("utf-8")

    provider = ProviderFactory.get_provider(provider_name)
    memberitems = MemberItems(provider, myredis)
    query_hash = memberitems.start_update(query)

    response_dict = {"query_hash": query_hash}
    resp = make_response(json.dumps(response_dict), 201)  # created
    resp.mimetype = "application/json"
    resp.headers["Access-Control-Allow-Origin"] = "*"
    return resp

def prep_collection_items(aliases):
    logger.info("got a list of aliases; creating new items for them.")
    try:
        # remove unprintable characters and change list to tuples
        clean_aliases = [(clean_id(namespace), clean_id(nid)) for [namespace, nid] in aliases]
    except ValueError:
        logger.error(
            "bad input to POST /collection (requires [namespace, id] pairs):{input}".format(input=str(clean_aliases))
        )
        abort(404, "POST /collection requires a list of [namespace, id] pairs.")

    logger.debug(
        "POST /collection got list of aliases; creating new items for {aliases}".format(aliases=str(clean_aliases))
    )

    (tiids, items) = create_or_find_items_from_aliases(clean_aliases)

    logger.debug("POST /collection saving a group of {num} new items: {items}".format(num=len(items), items=str(items)))

    # batch upload the new docs to the db
    # make sure they are there before the provider updates start happening
    for doc in mydao.db.update(items):
        pass

    # for each item, set the number of providers that need to run before the update is done
    # and put them on the update queue
    for item in items:
        myredis.set_num_providers_left(
            item["_id"], ProviderFactory.num_providers_with_metrics(default_settings.PROVIDERS)
        )
        myredis.add_to_alias_queue(item["_id"], item["aliases"])

    return tiids

def collection_update(cid=""):

    # first, get the tiids in this collection:
    try:
        collection = mydao.get(cid)
        tiids = collection["alias_tiids"].values()
    except Exception:
        logger.exception("couldn't get tiids for collection '{cid}'".format(cid=cid))
        abort(404, "couldn't get tiids for this collection...maybe doesn't exist?")

    # put each of them on the update queue
    for tiid in tiids:
        logger.debug("In update_item with tiid " + tiid)

        # set this so we know when it's still updating later on
        myredis.set_num_providers_left(tiid, ProviderFactory.num_providers_with_metrics(default_settings.PROVIDERS))

        item_doc = mydao.get(tiid)
        try:
            myredis.add_to_alias_queue(item_doc["_id"], item_doc["aliases"])
        except (KeyError, TypeError):
            logger.debug("couldn't get item_doc for {tiid}. Skipping its update".format(tiid=tiid))
            pass

    resp = make_response("true", 200)
    resp.mimetype = "application/json"
    return resp

def provider_method_wrapper(tiid, provider, method_name):

    # logger.info(u"{:20}: in provider_method_wrapper with {tiid} {provider_name} {method_name} with {aliases}".format(
    #    "wrapper", tiid=tiid, provider_name=provider.provider_name, method_name=method_name, aliases=input_aliases_dict))


    product = Product.query.get(tiid)

    if not product:
        logger.warning(u"Empty product in provider_run for tiid {tiid}".format(
           tiid=tiid))
        return None

    input_alias_tuples = product.aliases_for_providers
    try:
        method = getattr(provider, method_name)
    except AttributeError:
        provider = ProviderFactory.get_provider(provider)
        method = getattr(provider, method_name)

    provider_name = provider.provider_name
    worker_name = provider_name+"_worker"

    try:
        method_response = method(input_alias_tuples)
    except ProviderError, e:
        method_response = None

        logger.info(u"{:20}: **ProviderError {tiid} {method_name} {provider_name}, Exception type {exception_type} {exception_arguments}".format(
            worker_name, 
            tiid=tiid, 
            provider_name=provider_name.upper(), 
            method_name=method_name.upper(), 
            exception_type=type(e).__name__, 
            exception_arguments=e.args))

def get_metric_names(providers_config):
    full_metric_names = []
    providers = ProviderFactory.get_providers(providers_config)
    for provider in providers:
        metric_names = provider.metric_names()
        for metric_name in metric_names:
            full_metric_names.append(provider.provider_name + ':' + metric_name)
    return full_metric_names

 def test_03_init_aliases(self):
     providers = ProviderFactory.get_providers(self.config)
     pat = ProvidersAliasThread(providers, self.d)
     
     assert hasattr(pat, "stop")
     assert hasattr(pat, "stopped")
     assert hasattr(pat, "first")
     assert pat.queue is not None

def provider_memberitems(provider_name):
    """
    Make a descr string (like bibtex) into a dict strings describing items.
    """

    provider = ProviderFactory.get_provider(provider_name)
    items_dict = provider.parse(request.json["descr"])

    resp = make_response(
        json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4),
        200
    )
    return resp

def provider_biblio(provider_name, id):

    provider = ProviderFactory.get_provider(provider_name)
    if id=="example":
        id = provider.example_id[1]
        url = "http://localhost:8080/" + provider_name + "/biblio?%s"
    else:
        url = None

    biblio = provider.get_biblio_for_id(id, url, cache_enabled=False)
    resp = make_response( json.dumps(biblio, sort_keys=True, indent=4) )
    resp.mimetype = "application/json"
    return resp

def provider_memberitems(provider_name):
    query = request.values.get('query','')

    logger.debug("In provider_memberitems with " + query)

    provider = ProviderFactory.get_provider(provider_name)
    logger.debug("provider: " + provider.provider_name)

    memberitems = provider.member_items(query, cache_enabled=False)
    
    resp = make_response( json.dumps(memberitems, sort_keys=True, indent=4), 200 )
    resp.mimetype = "application/json"
    return resp

    def test_alias_queue(self):
        self.d.create_new_db_and_connect(self.testing_db_name)

        providers = ProviderFactory.get_providers(self.app.config["PROVIDERS"])

        response = self.client.post('/item/doi/' + quote_plus(TEST_DRYAD_DOI))
        tiid = json.loads(response.data)


        # now get it back out
        response = self.client.get('/item/' + tiid)
        print tiid
        assert_equals(response.status_code, 200)
        
        resp_dict = json.loads(response.data)
        assert_equals(
            set(resp_dict.keys()),
            set([u'tiid', u'created', u'last_requested', u'metrics', 
                u'last_modified', u'biblio', u'id', u'aliases'])
            )
        assert_equals(unicode(TEST_DRYAD_DOI), resp_dict["aliases"]["doi"][0])

        # test the view works
        res = self.d.view("aliases")
        assert len(res["rows"]) == 1, res
        assert_equals(TEST_DRYAD_DOI, res["rows"][0]["value"]["aliases"]["doi"][0])

        # see if the item is on the queue
        my_alias_queue = AliasQueue(self.d)
        assert isinstance(my_alias_queue.queue, list)
        assert_equals(len(my_alias_queue.queue), 1)
        
        # get our item from the queue
        my_item = my_alias_queue.first()
        assert_equals(my_item.aliases.doi[0], TEST_DRYAD_DOI)

        # do the update using the backend
        alias_thread = ProvidersAliasThread(providers, self.d)
        alias_thread.run(run_only_once=True)

        # get the item back out again and bask in the awesome
        response = self.client.get('/item/' + tiid)
        resp_dict = json.loads(response.data)
        print tiid
        print response.data
        assert_equals(
            resp_dict["aliases"]["title"][0],
            "data from: can clone size serve as a proxy for clone age? an exploration using microsatellite divergence in populus tremuloides"
            )
        print resp_dict
        assert_equals(resp_dict["biblio"]["data"]["year"], "2010")

    def setUp(self):
        #setup api test client
        self.app = api.app
        self.app.testing = True 
        self.client = self.app.test_client()

        # setup the database
        self.testing_db_name = "metrics_queue_test"
        self.old_db_name = self.app.config["DB_NAME"]
        self.app.config["DB_NAME"] = self.testing_db_name
        self.d = dao.Dao(self.testing_db_name, self.app.config["DB_URL"],
            self.app.config["DB_USERNAME"], self.app.config["DB_PASSWORD"])

        self.providers = ProviderFactory.get_providers(self.app.config["PROVIDERS"])

 def test_get_all_metric_names(self):
     response = ProviderFactory.get_all_metric_names(self.TEST_PROVIDER_CONFIG)
     expected = [
         "wikipedia:mentions",
         "mendeley:country",
         "pubmed:pmc_citations_reviews",
         "mendeley:discipline",
         "pubmed:f1000",
         "mendeley:career_stage",
         "pubmed:pmc_citations_editorials",
         "mendeley:readers",
         "pubmed:pmc_citations",
         "mendeley:groups",
     ]
     assert_equals(response, expected)

def get_metric_value_lists(items):
    (ordered_fieldnames, rows) = make_csv_rows(items)
    metric_values = {}
    for metric_name in ProviderFactory.get_all_metric_names():
        if metric_name in ordered_fieldnames:
            if metric_name in ["tiid", "title", "doi"]:
                pass
            else:
                values = [row[metric_name] for row in rows]
                values = [value if value else 0 for value in values]
                # treat "Yes" as 1 for normalizaations
                values = [1 if value=="Yes" else value for value in values]
                metric_values[metric_name] = sorted(values, reverse=True)
        else:
            metric_values[metric_name] = [0 for row in rows]
    return metric_values