def reports_node(env, node_name, page):
    """Fetches all reports for a node and processes them eventually rendering
    a table displaying those reports.

    :param env: Search for reports in this environment
    :type env: :obj:`string`
    :param node_name: Find the reports whose certname match this value
    :type node_name: :obj:`string`
    :param page: Calculates the offset of the query based on the report count
        and this value
    :type page: :obj:`int`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        query = '["=", "certname", "{0}"]]'.format(node_name)
    else:
        query='["and",' \
            '["=", "environment", "{0}"],' \
            '["=", "certname", "{1}"]]'.format(env, node_name),

    reports = get_or_abort(puppetdb.reports,
        query=query,
        limit=app.config['REPORTS_COUNT'],
        offset=(page-1) * app.config['REPORTS_COUNT'],
        order_by='[{"field": "start_time", "order": "desc"}]')
    total = get_or_abort(puppetdb._query,
        'reports',
        query='["extract", [["function", "count"]],' \
            '["and", ["=", "environment", "{0}"], ["=", "certname", "{1}"]]]'.format(
            env,
            node_name))
    total = total[0]['count']
    reports, reports_events = tee(reports)
    report_event_counts = {}

    if total == 0 and page != 1:
        abort(404)

    for report in reports_events:
        counts = get_or_abort(puppetdb.event_counts,
            query='["and",' \
                '["=", "environment", "{0}"],' \
                '["=", "certname", "{1}"],' \
                '["=", "report", "{2}"]]'.format(env, report.node, report.hash_),
            summarize_by="certname")
        try:
            report_event_counts[report.hash_] = counts[0]
        except IndexError:
            report_event_counts[report.hash_] = {}
    return render_template(
        'reports.html',
        reports=reports,
        reports_count=app.config['REPORTS_COUNT'],
        report_event_counts=report_event_counts,
        pagination=Pagination(page, app.config['REPORTS_COUNT'], total),
        envs=envs,
        current_env=env)

def node(env, node_name):
    """Display a dashboard for a node showing as much data as we have on that
    node. This includes facts and reports but not Resources as that is too
    heavy to do within a single request.

    :param env: Ensure that the node, facts and reports are in this environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        query = '["=", "certname", "{0}"]]'.format(node_name)
    else:
        query='["and", ["=", "environment", "{0}"],' \
            '["=", "certname", "{1}"]]'.format(env, node_name),

    node = get_or_abort(puppetdb.node, node_name)
    facts = node.facts()
    reports = get_or_abort(puppetdb.reports,
        query=query,
        limit=app.config['REPORTS_COUNT'],
        order_by='[{"field": "start_time", "order": "desc"}]')
    reports, reports_events = tee(reports)
    report_event_counts = {}

    for report in reports_events:
        report_event_counts[report.hash_] = {}

        for event in report.events():
            if event.status == 'success':
                try:
                    report_event_counts[report.hash_]['successes'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['successes'] = 1
            elif event.status == 'failure':
                try:
                    report_event_counts[report.hash_]['failures'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['failures'] = 1
            elif event.status == 'noop':
                try:
                    report_event_counts[report.hash_]['noops'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['noops'] = 1
            elif event.status == 'skipped':
                try:
                    report_event_counts[report.hash_]['skips'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['skips'] = 1
    return render_template(
        'node.html',
        node=node,
        facts=yield_or_stop(facts),
        reports=yield_or_stop(reports),
        reports_count=app.config['REPORTS_COUNT'],
        report_event_counts=report_event_counts,
        envs=envs,
        current_env=env)

def reports(env, page):
    """Displays a list of reports and status from all nodes, retreived using the
    reports endpoint, sorted by start_time.

    :param env: Search for all reports in this environment
    :type env: :obj:`string`
    :param page: Calculates the offset of the query based on the report count
        and this value
    :type page: :obj:`int`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        reports_query = None
        total_query = '["extract", [["function", "count"]], ["~", "certname", ""]]'
    else:
        reports_query = '["=", "environment", "{0}"]'.format(env)
        total_query = '["extract", [["function", "count"]],'\
            '["and", ["=", "environment", "{0}"]]]'.format(env)

    reports = get_or_abort(puppetdb.reports,
        query=reports_query,
        limit=app.config['REPORTS_COUNT'],
        offset=(page-1) * app.config['REPORTS_COUNT'],
        order_by='[{"field": "start_time", "order": "desc"}]')
    total = get_or_abort(puppetdb._query,
        'reports',
        query=total_query)
    total = total[0]['count']
    reports, reports_events = tee(reports)
    report_event_counts = {}

    if total == 0 and page != 1:
        abort(404)

    for report in reports_events:
        counts = get_or_abort(puppetdb.event_counts,
            query='["and",' \
                '["=", "environment", "{0}"],' \
                '["=", "certname", "{1}"],' \
                '["=", "report", "{2}"]]'.format(
                    env,
                    report.node,
                    report.hash_),
            summarize_by="certname")
        try:
            report_event_counts[report.hash_] = counts[0]
        except IndexError:
            report_event_counts[report.hash_] = {}
    return Response(stream_with_context(stream_template(
        'reports.html',
        reports=yield_or_stop(reports),
        reports_count=app.config['REPORTS_COUNT'],
        report_event_counts=report_event_counts,
        pagination=Pagination(page, app.config['REPORTS_COUNT'], total),
        envs=envs,
        current_env=env)))

def index():
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.
    """
    # TODO: Would be great if we could parallelize this somehow, doing these
    # requests in sequence is rather pointless.
    prefix = 'com.puppetlabs.puppetdb.query.population'
    num_nodes = get_or_abort(
        puppetdb.metric,
        "{0}{1}".format(prefix, ':type=default,name=num-nodes'))
    num_resources = get_or_abort(
        puppetdb.metric,
        "{0}{1}".format(prefix, ':type=default,name=num-resources'))
    avg_resources_node = get_or_abort(
        puppetdb.metric,
        "{0}{1}".format(prefix, ':type=default,name=avg-resources-per-node'))
    metrics = {
        'num_nodes': num_nodes['Value'],
        'num_resources': num_resources['Value'],
        'avg_resources_node': "{0:10.0f}".format(avg_resources_node['Value']),
        }

    nodes = puppetdb.nodes(
        unreported=app.config['UNRESPONSIVE_HOURS'],
        with_status=True)

    nodes_overview = []
    stats = {
        'changed': 0,
        'unchanged': 0,
        'failed': 0,
        'unreported': 0,
        'noop': 0
        }

    for node in nodes:
        if node.status == 'unreported':
            stats['unreported'] += 1
        elif node.status == 'changed':
            stats['changed'] += 1
        elif node.status == 'failed':
            stats['failed'] += 1
        elif node.status == 'noop':
            stats['noop'] += 1
        else:
            stats['unchanged'] += 1

        if node.status != 'unchanged':
            nodes_overview.append(node)

    return render_template(
        'index.html',
        metrics=metrics,
        nodes=nodes_overview,
        stats=stats
        )

def index(env):
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.

    :param env: Search for nodes in this (Catalog and Fact) environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    stats = status_count(env)
    if stats['total'] < 1000:
        stats['num_resources'] = get_count(
            'resources', EqualsOperator("environment", env))

        # average resource / node
        try:
            stats['avg_resources_node'] = "{0:10.0f}".format(
                (stats['num_resources'] / stats['total']))
        except ZeroDivisionError:
            stats['avg_resources_node'] = 0

    else:
        prefix = 'puppetlabs.puppetdb.population'
        query_type = ''

        # Puppet DB version changed the query format from 3.2.0
        # to 4.0 when querying mbeans
        if get_db_version(puppetdb) < (4, 0, 0):
            query_type = 'type=default,'

        stats['num_resources'] = get_or_abort(
            puppetdb.metric, "{0}{1}".format(
                prefix, ':%sname=num-resources' % query_type))['Value']
        stats['avg_resources_node'] = int(get_or_abort(
            puppetdb.metric, "{0}{1}".format(
                prefix,
                ':%sname=avg-resources-per-node' % query_type))['Value'])

    paging_args = {'limit': app.config['NORMAL_TABLE_COUNT'], 'offset': 0}
    order_arg = '[{"field": "catalog_timestamp", "order": "desc"}]'
    nodes = get_or_abort(puppetdb.nodes,
                         query=get_node_env_query(env),
                         unreported=app.config['UNRESPONSIVE_HOURS'],
                         with_status=True,
                         order_by=order_arg,
                         **paging_args)

    return render_template(
        'index.html',
        nodes=nodes,
        stats=stats,
        envs=envs,
        current_env=env
    )

def catalog_compare(compare, against):
    """Compares the catalog of one node, parameter compare, with that of
       with that of another node, parameter against.
    """
    if app.config["ENABLE_CATALOG"]:
        compare_cat = get_or_abort(puppetdb.catalog, node=compare)
        against_cat = get_or_abort(puppetdb.catalog, node=against)

        return render_template("catalog_compare.html", compare=compare_cat, against=against_cat)
    else:
        log.warn("Access to catalog interface disabled by administrator")
        abort(403)

def test_http_connection_error(mock_log):
    err = "ConnectionError"

    def connection_error():
        x = Response()
        x.status_code = 500
        x.reason = err
        raise ConnectionError(err, response=x)

    with pytest.raises(InternalServerError):
        utils.get_or_abort(connection_error)
        mock_log.error.assert_called_with(err)

def test_http_error(mock_log):
    err = "NotFound"

    def raise_http_error():
        x = Response()
        x.status_code = 404
        x.reason = err
        raise HTTPError(err, response=x)

    with pytest.raises(NotFound):
        utils.get_or_abort(raise_http_error)
        mock_log.error.assert_called_once_with(err)

def report_latest(node_name):
    """Redirect to the latest report of a given node. This is a workaround
    as long as PuppetDB can't filter reports for latest-report? field. This
    feature has been requested: http://projects.puppetlabs.com/issues/21554
    """
    node = get_or_abort(puppetdb.node, node_name)
    reports = get_or_abort(puppetdb._query, "reports", query='["=","certname","{0}"]'.format(node_name), limit=1)
    if len(reports) > 0:
        report = reports[0]["hash"]
        return redirect(url_for("report", node=node_name, report_id=report))
    else:
        abort(404)

def report_latest(node_name):
    """Redirect to the latest report of a given node. This is a workaround
    as long as PuppetDB can't filter reports for latest-report? field. This
    feature has been requested: https://tickets.puppetlabs.com/browse/PDB-203
    """
    node = get_or_abort(puppetdb.node, node_name)
    reports = get_or_abort(puppetdb._query, 'reports',
                           query='["=","certname","{0}"]'.format(node_name),
                           limit=1)
    if len(reports) > 0:
        report = reports[0]['hash']
        return redirect(url_for('report', node=node_name, report_id=report))
    else:
        abort(404)

def report_latest(env, node_name):
    """Redirect to the latest report of a given node.

    :param env: Search for reports in this environment
    :type env: :obj:`string`
    :param node_name: Find the reports whose certname match this value
    :type node_name: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        node_query = '["=", "certname", "{0}"]'.format(node_name)
    else:
        node_query = '["and",' \
            '["=", "report_environment", "{0}"],' \
            '["=", "certname", "{1}"]]'.format(env, node_name)

    try:
        node = next(get_or_abort(puppetdb.nodes,
            query=node_query,
            with_status=True))
    except StopIteration:
        abort(404)

    if node.latest_report_hash is not None:
        hash_ = node.latest_report_hash
    else:
        if env == '*':
            query='["and",' \
                '["=", "certname", "{0}"],' \
                '["=", "latest_report?", true]]'.format(node.name)
        else:
            query='["and",' \
                '["=", "environment", "{0}"],' \
                '["=", "certname", "{1}"],' \
                '["=", "latest_report?", true]]'.format(
                    env,
                    node.name)

        reports = get_or_abort(puppetdb.reports, query=query)
        try:
            report = next(reports)
            hash_ = report.hash_
        except StopIteration:
            abort(404)

    return redirect(
        url_for('report', env=env, node_name=node_name, report_id=hash_))

def node(env, node_name):
    """Display a dashboard for a node showing as much data as we have on that
    node. This includes facts and reports but not Resources as that is too
    heavy to do within a single request.

    :param env: Ensure that the node, facts and reports are in this environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)
    query = AndOperator()

    if env != '*':
        query.add(EqualsOperator("environment", env))

    query.add(EqualsOperator("certname", node_name))

    node = get_or_abort(puppetdb.node, node_name)

    return render_template(
        'node.html',
        node=node,
        envs=envs,
        current_env=env,
        columns=REPORTS_COLUMNS[:2])

def test_get():
    x = "hello world"

    def test_get_or_abort():
        return x

    assert x == utils.get_or_abort(test_get_or_abort)

def facts(env):
    """Displays an alphabetical list of all facts currently known to
    PuppetDB.

    :param env: Serves no purpose for this function, only for consistency's
        sake
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    facts_dict = collections.defaultdict(list)
    facts = get_or_abort(puppetdb.fact_names)
    for fact in facts:
        letter = fact[0].upper()
        letter_list = facts_dict[letter]
        letter_list.append(fact)
        facts_dict[letter] = letter_list

    sorted_facts_dict = sorted(facts_dict.items())
    return render_template(
        "facts.html",
        facts_dict=sorted_facts_dict,
        facts_len=(sum(map(len, facts_dict.values())) + len(facts_dict) * 5),
        envs=envs,
        current_env=env,
    )

def catalogs():
    if app.config["ENABLE_CATALOG"]:
        nodenames = []
        catalog_list = []
        nodes = get_or_abort(
            puppetdb.nodes,
            query='["null?", "catalog_timestamp", false]',
            with_status=False,
            order_by='[{"field": "certname", "order": "asc"}]',
        )
        nodes, temp = tee(nodes)

        for node in temp:
            nodenames.append(node.name)

        for node in nodes:
            table_row = {"name": node.name, "catalog_timestamp": node.catalog_timestamp}

            if len(nodenames) > 1:
                form = CatalogForm()

                form.compare.data = node.name
                form.against.choices = [(x, x) for x in nodenames if x != node.name]
                table_row["form"] = form
            else:
                table_row["form"] = None

            catalog_list.append(table_row)

        return render_template("catalogs.html", nodes=catalog_list)
    else:
        log.warn("Access to catalogs endpoint disabled by administrator")
        abort(403)

def fact_value(env, fact, value):
    """On asking for fact/value get all nodes with that fact.

    :param env: Searches for facts in this environment
    :type env: :obj:`string`
    :param fact: Find all facts with this name
    :type fact: :obj:`string`
    :param value: Filter facts whose value is equal to this
    :type value: :obj:`string`
    """
    check_env(env)

    if env == '*':
        query = None
    else:
        query = '["=", "environment", "{0}"]'.format(env)
    facts = get_or_abort(puppetdb.facts,
        name=fact,
        value=value,
        query=query)
    localfacts = [f for f in yield_or_stop(facts)]
    return render_template(
        'fact.html',
        name=fact,
        value=value,
        facts=localfacts,
        envs=envs,
        current_env=env)

def query(env):
    """Allows to execute raw, user created querries against PuppetDB. This is
    currently highly experimental and explodes in interesting ways since none
    of the possible exceptions are being handled just yet. This will return
    the JSON of the response or a message telling you what whent wrong /
    why nothing was returned.

    :param env: Serves no purpose for the query data but is required for the
        select field in the environment block
    :type env: :obj:`string`
    """
    if app.config["ENABLE_QUERY"]:
        envs = environments()
        check_env(env, envs)

        form = QueryForm(meta={"csrf_secret": app.config["SECRET_KEY"], "csrf_context": session})
        if form.validate_on_submit():
            if form.endpoints.data == "pql":
                query = form.query.data
            elif form.query.data[0] == "[":
                query = form.query.data
            else:
                query = "[{0}]".format(form.query.data)
            result = get_or_abort(puppetdb._query, form.endpoints.data, query=query)
            return render_template("query.html", form=form, result=result, envs=envs, current_env=env)
        return render_template("query.html", form=form, envs=envs, current_env=env)
    else:
        log.warn("Access to query interface disabled by administrator..")
        abort(403)

def environments():
    envs = get_or_abort(puppetdb.environments)
    x = []

    for env in envs:
        x.append(env["name"])

    return x

def node(node_name):
    """Display a dashboard for a node showing as much data as we have on that
    node. This includes facts and reports but not Resources as that is too
    heavy to do within a single request.
    """
    node = get_or_abort(puppetdb.node, node_name)
    facts = node.facts()
    reports = ten_reports(node.reports())
    return render_template("node.html", node=node, facts=yield_or_stop(facts), reports=yield_or_stop(reports))

def get_count(endpoint, query):
    c_query = ExtractOperator()
    c_query.add_field(FunctionOperator('count'))
    if query:
        c_query.add_query(query)
    res = get_or_abort(
        puppetdb._query, endpoint,
        query=c_query)
    return res[0]['count']