def test_cluster(self):
        uptime = "22h 41m 49s"
        cluster = {
            "stormVersion":"0.9.3",
            "nimbusUptime": uptime,
            "supervisors":7,
            "slotsTotal":147,
            "slotsUsed":7,
            "slotsFree":140,
            "executorsTotal":11415,
            "tasksTotal":11415
        }
        instance = {'url': 'http://localhost:8080', 'timeout': 0, 'tags': ['cluster_want:form'], "cache_file": "/dev/null"}
        conf = {
            'init_config': {},
            'instances': [instance]
        }
        self.check = load_check('storm_rest_api', conf, {})
        self.check.report_cluster(self.check.instance_config(instance), cluster)
        metrics = self.check.get_metrics()

        cluster_uptime = self.find_metric(metrics, 'storm.rest.cluster.nimbus_uptime_seconds')
        self.assertEqual(81709, cluster_uptime[2])
        self.assert_tags(['cluster_want:form'], cluster_uptime[3]['tags'])

        cluster_slots_used = self.find_metric(metrics, 'storm.rest.cluster.slots_used_count')
        self.assertEqual(7, cluster_slots_used[2])
        print cluster_slots_used
        self.assert_tags(['cluster_want:form'], cluster_slots_used[3]['tags'])

        for metric_name in ['supervisor_count', 'slots_total_count', 'slots_free_count', 'executors_total_count', 'tasks_total_count']:
            metric = self.find_metric(metrics, 'storm.rest.cluster.%s' % metric_name)
            self.assert_tags(['cluster_want:form'], metric[3]['tags'])
            self.assertTrue(metric[2] > 0)

 def test_nginx_plus(self):
     test_data = Fixtures.read_file('nginx_plus_in.json')
     expected = eval(Fixtures.read_file('nginx_plus_out.python'))
     nginx = load_check('nginx', self.config, self.agent_config)
     parsed = nginx.parse_json(test_data)
     parsed.sort()
     self.assertEquals(parsed, expected)

    def test_topology_details(self):
        topology_from_sum = {"id":"sometopo_987IENien9887a-3-1464117779","encodedId":"sometopo_987IENien9887a-3-1464117779","name":"sometopo_987IENien9887a","status":"ACTIVE","uptime":"30s","tasksTotal":11366,"workersTotal":7,"executorsTotal":11366}
        name = 'sometopo'
        topology_details = {
            "spouts": [
                {"executors": 48, "emitted": 0, "errorLapsedSecs": 13822, "completeLatency": "2.030", "transferred": 50, "acked": 40, "errorPort": 6711, "spoutId": "somespout", "tasks": 48, "errorHost": "", "lastError": "", "errorWorkerLogLink": "", "failed": 20, "encodedSpoutId": "somespout"},
                {"executors": 48, "emitted": 0, "errorLapsedSecs": 13822, "completeLatency": "2.030", "transferred": 50, "acked": 40, "errorPort": 6711, "spoutId": "detailspout", "tasks": 48, "errorHost": "", "lastError": "", "errorWorkerLogLink": "", "failed": 20, "encodedSpoutId": "detailspout"}
            ],
            "bolts": [
                {"executors": 3, "emitted": 10, "errorLapsedSecs": None, "transferred": 12, "acked": 9, "errorPort": "", "executeLatency": "2.300", "tasks": 4, "executed": 12, "processLatency": "2.501", "boltId": "somebolt", "errorHost": "", "lastError": "", "errorWorkerLogLink": "", "capacity": "0.020", "failed": 2, "encodedBoltId": "somebolt"},
                {"executors": 3, "emitted": 10, "errorLapsedSecs": None, "transferred": 12, "acked": 3, "errorPort": "", "executeLatency": "2.300", "tasks": 4, "executed": 12, "processLatency": "2.501", "boltId": "detail::bolt", "errorHost": "", "lastError": "", "errorWorkerLogLink": "", "capacity": "0.020", "failed": 2, "encodedBoltId": "detail%3A%3Abolt"}

            ]
        }
        instance = {
            'url': 'http://localhost:8080',
            'timeout': 0,
            'topologies': '^(sometopo)_.*$',
            'executor_details_whitelist': ['detailspout', 'detail::bolt'],
            'task_tags': {
                'spout': {
                    'somespout': [
                        'is_a_great_spout:true'
                    ]
                },
                'bolt': {
                    'somebolt': [
                        'is_a_great_bolt:true'
                    ]
                }
            },
            "cache_file": "/dev/null"
        }
        conf = {
            'init_config': {},
            'instances': [
                instance
            ],
        }
        self.check = load_check('storm_rest_api', conf, {})

        self.check.report_topology(self.check.instance_config(instance), name, topology_details)
        self.check.report_topology(self.check.instance_config(instance), name, topology_details)

        metrics = self.check.get_metrics()
        spout_workers_metric = self.find_metric(metrics, 'storm.rest.spout.executors_total', ['storm_task_id:somespout'])
        self.assertEqual(48, spout_workers_metric[2])
        print spout_workers_metric[3]
        self.assert_tags(['storm_topology:sometopo', 'storm_task_id:somespout', 'is_a_great_spout:true'], spout_workers_metric[3]['tags'])

        complete_latency_metric = self.find_metric(metrics, 'storm.rest.spout.complete_latency_us', ['storm_task_id:somespout'])
        self.assertEqual(2.030, complete_latency_metric[2])

        bolt_workers_metric = self.find_metric(metrics, 'storm.rest.bolt.executors_total', ['storm_task_id:somebolt'])
        self.assertEqual(3, bolt_workers_metric[2])
        self.assert_tags(['storm_topology:sometopo', 'storm_task_id:somebolt', 'is_a_great_bolt:true'], bolt_workers_metric[3]['tags'])

        bolt_executed_metric = self.find_metric(metrics, 'storm.rest.bolt.executed_total', ['storm_task_id:somebolt'])
        self.assertEqual(0, bolt_executed_metric[2])
        self.assert_tags(['storm_topology:sometopo', 'storm_task_id:somebolt', 'is_a_great_bolt:true'], bolt_workers_metric[3]['tags'])

    def test_no_profiling(self):
        agentConfig = {
            'api_key': 'XXXtest_apikey',
            'developer_mode': True,
            'allow_profiling': False
        }
        # this must be SystemExit, because otherwise the Exception is eaten
        mocks = {
            '_set_internal_profiling_stats': mock.MagicMock(side_effect=SystemExit),
        }
        redis_config = {
            "init_config": {},
            "instances": [{"host": "localhost", "port": 6379}]
        }
        check = load_check('redisdb', redis_config, agentConfig)

        self.assertFalse(check.allow_profiling)
        self.assertTrue(check.in_developer_mode)

        for func_name, mock1 in mocks.iteritems():
            if not hasattr(check, func_name):
                continue
            else:
                setattr(check, func_name, mock1)

        check.run()

    def test_apptags(self):
        '''
        Tests that the app tags are sent if specified so
        '''
        agentConfig = {
            'agent_key': 'test_agentkey',
            'collect_ec2_tags': False,
            'collect_orchestrator_tags': False,
            'collect_instance_metadata': False,
            'create_sd_check_tags': True,
            'version': 'test',
            'tags': '',
        }

        # Run a single checks.d check as part of the collector.
        disk_config = {
            "init_config": {},
            "instances": [{}]
        }

        checks = [load_check('disk', disk_config, agentConfig)]

        c = Collector(agentConfig, [], {}, get_hostname(agentConfig))
        payload = c.run({
            'initialized_checks': checks,
            'init_failed_checks': {}
        })

        # We check that the redis SD_CHECK_TAG is sent in the payload
        self.assertTrue('sd_check:disk' in payload['host-tags']['system'])

    def test_config_parser(self):
        check = load_check(self.CHECK_NAME, {}, {})
        instance = {
            "username": "user",
            "password": "pass",
            "is_external": "yes",
            "url": "http://foo.bar",
            "tags": ["a", "b:c"],
        }

        c = check.get_instance_config(instance)
        self.assertEquals(c.username, "user")
        self.assertEquals(c.password, "pass")
        self.assertEquals(c.cluster_stats, True)
        self.assertEquals(c.url, "http://foo.bar")
        self.assertEquals(c.tags, ["url:http://foo.bar", "a", "b:c"])
        self.assertEquals(c.timeout, check.DEFAULT_TIMEOUT)
        self.assertEquals(c.service_check_tags, ["host:foo.bar", "port:None"])

        instance = {
            "url": "http://192.168.42.42:12999",
            "timeout": 15
        }

        c = check.get_instance_config(instance)
        self.assertEquals(c.username, None)
        self.assertEquals(c.password, None)
        self.assertEquals(c.cluster_stats, False)
        self.assertEquals(c.url, "http://192.168.42.42:12999")
        self.assertEquals(c.tags, ["url:http://192.168.42.42:12999"])
        self.assertEquals(c.timeout, 15)
        self.assertEquals(c.service_check_tags,
                          ["host:192.168.42.42", "port:12999"])

    def test_register_psutil_metrics(self):
        check = load_check(self.CHECK_NAME, MOCK_CONFIG, AGENT_CONFIG_DEV_MODE)
        check._register_psutil_metrics(MOCK_STATS, MOCK_NAMES_TO_METRIC_TYPES)
        self.metrics = check.get_metrics()

        self.assertMetric('datadog.agent.collector.memory_info.rss', value=16814080)
        self.assertMetric('datadog.agent.collector.memory_info.vms', value=74522624)

    def test_redis_repl(self):
        master_instance = {
            'host': 'localhost',
            'port': NOAUTH_PORT
        }

        slave_instance = {
            'host': 'localhost',
            'port': AUTH_PORT,
            'password': 'datadog-is-devops-best-friend'
        }

        repl_metrics = [
            'redis.replication.delay',
            'redis.replication.backlog_histlen',
            'redis.replication.delay',
            'redis.replication.master_repl_offset',
        ]

        master_db = redis.Redis(port=NOAUTH_PORT, db=14)
        slave_db = redis.Redis(port=AUTH_PORT, password=slave_instance['password'], db=14)
        master_db.flushdb()

        # Assert that the replication works
        master_db.set('replicated:test', 'true')
        self.assertEquals(slave_db.get('replicated:test'), 'true')

        r = load_check('redisdb', {}, {})
        r.check(master_instance)
        metrics = self._sort_metrics(r.get_metrics())

        # Assert the presence of replication metrics
        keys = [m[0] for m in metrics]
        assert [x in keys for x in repl_metrics]

    def test_network_latency_checks(self):
        self.check = load_check(self.CHECK_NAME, MOCK_CONFIG_NETWORK_LATENCY_CHECKS,
                                self.DEFAULT_AGENT_CONFIG)

        mocks = self._get_consul_mocks()

        # We start out as the leader, and stay that way
        instance_hash = hash_mutable(MOCK_CONFIG_NETWORK_LATENCY_CHECKS['instances'][0])
        self.check._instance_states[instance_hash].last_known_leader = self.mock_get_cluster_leader_A(None)

        self.run_check(MOCK_CONFIG_NETWORK_LATENCY_CHECKS, mocks=mocks)

        latency = [m for m in self.metrics if m[0].startswith('consul.net.')]
        latency.sort()
        # Make sure we have the expected number of metrics
        self.assertEquals(19, len(latency))

        # Only 3 dc-latency metrics since we only do source = self
        dc = [m for m in latency if '.dc.latency.' in m[0]]
        self.assertEquals(3, len(dc))
        self.assertEquals(1.6746410750238774, dc[0][2])

        # 16 latency metrics, 2 nodes * 8 metrics each
        node = [m for m in latency if '.node.latency.' in m[0]]
        self.assertEquals(16, len(node))
        self.assertEquals(0.26577747932995816, node[0][2])

    def test_build_event(self):
        agent_config = {
            'version': '0.1',
            'api_key': 'toto'
        }
        check = load_check('teamcity', CONFIG, agent_config)

        with patch('requests.get', get_mock_first_build):
            check.check(check.instances[0])

        metrics = check.get_metrics()
        self.assertEquals(len(metrics), 0)

        events = check.get_events()
        # Nothing should have happened because we only create events
        # for newer builds
        self.assertEquals(len(events), 0)

        with patch('requests.get', get_mock_one_more_build):
            check.check(check.instances[0])

        events = check.get_events()
        self.assertEquals(len(events), 1)
        self.assertEquals(events[0]['msg_title'], "Build for One test build successful")
        self.assertEquals(events[0]['msg_text'], "Build Number: 2\nDeployed To: buildhost42.dtdg.co\n\nMore Info: http://localhost:8111/viewLog.html?buildId=2&buildTypeId=TestProject_TestBuild")
        self.assertEquals(events[0]['tags'], ['build', 'one:tag', 'one:test'])
        self.assertEquals(events[0]['host'], "buildhost42.dtdg.co")


        # One more check should not create any more events
        with patch('requests.get', get_mock_one_more_build):
            check.check(check.instances[0])

        events = check.get_events()
        self.assertEquals(len(events), 0)

    def test_cull_services_list(self):
        self.check = load_check(self.CHECK_NAME, MOCK_CONFIG_LEADER_CHECK, self.DEFAULT_AGENT_CONFIG)

        # Pad num_services to kick in truncation logic
        num_services = self.check.MAX_SERVICES + 20

        # Big whitelist
        services = self.mock_get_n_services_in_cluster(num_services)
        whitelist = ['service_{0}'.format(k) for k in range(num_services)]
        self.assertEqual(len(self.check._cull_services_list(services, whitelist)), self.check.MAX_SERVICES)

        # Whitelist < MAX_SERVICES should spit out the whitelist
        services = self.mock_get_n_services_in_cluster(num_services)
        whitelist = ['service_{0}'.format(k) for k in range(self.check.MAX_SERVICES-1)]
        self.assertEqual(set(self.check._cull_services_list(services, whitelist)), set(whitelist))

        # No whitelist, still triggers truncation
        whitelist = []
        self.assertEqual(len(self.check._cull_services_list(services, whitelist)), self.check.MAX_SERVICES)

        # Num. services < MAX_SERVICES should be no-op in absence of whitelist
        num_services = self.check.MAX_SERVICES - 1
        services = self.mock_get_n_services_in_cluster(num_services)
        self.assertEqual(len(self.check._cull_services_list(services, whitelist)), num_services)

        # Num. services < MAX_SERVICES should spit out only the whitelist when one is defined
        num_services = self.check.MAX_SERVICES - 1
        whitelist = ['service_1', 'service_2', 'service_3']
        services = self.mock_get_n_services_in_cluster(num_services)
        self.assertEqual(set(self.check._cull_services_list(services, whitelist)), set(whitelist))

    def test_apptags(self):
        '''
        Tests that the app tags are sent if specified so
        '''
        agentConfig = {
            'api_key': 'test_apikey',
            'collect_ec2_tags': False,
            'collect_instance_metadata': False,
            'create_dd_check_tags': True,
            'version': 'test',
            'tags': '',
        }

        # Run a single checks.d check as part of the collector.
        redis_config = {
            "init_config": {},
            "instances": [{"host": "localhost", "port": 6379}]
        }
        checks = [load_check('redisdb', redis_config, agentConfig)]

        c = Collector(agentConfig, [], {}, get_hostname(agentConfig))
        payload = c.run({
            'initialized_checks': checks,
            'init_failed_checks': {}
        })

        # We check that the redis DD_CHECK_TAG is sent in the payload
        self.assertTrue('dd_check:redisdb' in payload['host-tags']['system'])

    def init_check(self, config, check_name):
        self.agentConfig = {
            'version': AGENT_VERSION,
            'api_key': 'toto'
        }

        self.check = load_check(check_name, config, self.agentConfig)
        self.checks.append(self.check)

 def test_metric_name_without_prefix(self):
     instance = {'url': 'http://localhost:8080', 'timeout': 0, "cache_file": "/dev/null"}
     conf = {
         'init_config': {},
         'instances': [instance]
     }
     self.check = load_check('storm_rest_api', conf, {})
     self.assertEqual('storm.rest.baz', self.check.metric(self.check.instance_config(instance), 'baz'))

        def test_service_checks(self):
            self.check = load_check(self.CHECK_NAME, self.config, {})
            self.assertRaises(error, lambda: self.run_check(self.config))

            self.assertEqual(len(self.service_checks), 1, self.service_checks)
            self.assertServiceCheck(self.check.SERVICE_CHECK_NAME,
                                    status=AgentCheck.CRITICAL,
                                    tags=['aggregation_key:localhost:8080'])

    def test_cull_services_list(self):
        self.check = load_check(self.CHECK_NAME, MOCK_CONFIG_LEADER_CHECK, self.DEFAULT_AGENT_CONFIG)

        # Pad num_services to kick in truncation logic
        num_services = self.check.MAX_SERVICES + 20

        # Max services parameter (from consul.yaml) set to be bigger than MAX_SERVICES and smaller than the total of services
        max_services = num_services - 10

        # Big whitelist
        services = self.mock_get_n_services_in_cluster(num_services)
        whitelist = ['service_{0}'.format(k) for k in range(num_services)]
        self.assertEqual(len(self.check._cull_services_list(services, whitelist)), self.check.MAX_SERVICES)

        # Big whitelist with max_services
        services = self.mock_get_n_services_in_cluster(num_services)
        whitelist = ['service_{0}'.format(k) for k in range(num_services)]
        self.assertEqual(len(self.check._cull_services_list(services, whitelist, max_services)), max_services)

        # Whitelist < MAX_SERVICES should spit out the whitelist
        services = self.mock_get_n_services_in_cluster(num_services)
        whitelist = ['service_{0}'.format(k) for k in range(self.check.MAX_SERVICES-1)]
        self.assertEqual(set(self.check._cull_services_list(services, whitelist)), set(whitelist))

        # Whitelist < max_services param should spit out the whitelist
        services = self.mock_get_n_services_in_cluster(num_services)
        whitelist = ['service_{0}'.format(k) for k in range(max_services-1)]
        self.assertEqual(set(self.check._cull_services_list(services, whitelist, max_services)), set(whitelist))

        # No whitelist, still triggers truncation
        whitelist = []
        self.assertEqual(len(self.check._cull_services_list(services, whitelist)), self.check.MAX_SERVICES)

        # No whitelist with max_services set, also triggers truncation
        whitelist = []
        self.assertEqual(len(self.check._cull_services_list(services, whitelist, max_services)), max_services)

        # Num. services < MAX_SERVICES should be no-op in absence of whitelist
        num_services = self.check.MAX_SERVICES - 1
        services = self.mock_get_n_services_in_cluster(num_services)
        self.assertEqual(len(self.check._cull_services_list(services, whitelist)), num_services)

        # Num. services < max_services (from consul.yaml) should be no-op in absence of whitelist
        num_services = max_services - 1
        services = self.mock_get_n_services_in_cluster(num_services)
        self.assertEqual(len(self.check._cull_services_list(services, whitelist, max_services)), num_services)

        # Num. services < MAX_SERVICES should spit out only the whitelist when one is defined
        num_services = self.check.MAX_SERVICES - 1
        whitelist = ['service_1', 'service_2', 'service_3']
        services = self.mock_get_n_services_in_cluster(num_services)
        self.assertEqual(set(self.check._cull_services_list(services, whitelist)), set(whitelist))

        # Num. services < max_services should spit out only the whitelist when one is defined
        num_services = max_services - 1
        whitelist = ['service_1', 'service_2', 'service_3']
        services = self.mock_get_n_services_in_cluster(num_services)
        self.assertEqual(set(self.check._cull_services_list(services, whitelist, max_services)), set(whitelist))

 def __init__(self, *args, **kwargs):
     unittest.TestCase.__init__(self, *args, **kwargs)
     self.config = {"instances": [{
         "access_id":"foo",
         "access_secret": "bar"}]}
     self.check = load_check(self.CHECK_NAME, self.config, {})
     self.check._connect = Mock(return_value=(None, None, ["aggregation_key:localhost:8080"]))
     self.check._get_stats = Mock(return_value=self.check.load_json(
         Fixtures.read_file('riakcs_in.json')))

    def test_nginx_ssl_validation_enabled(self):
        # Note: Throws an SSLError, because we're attempting to connect to an https endpoint with a self-signed
        #       certificate. In addition, this throws an InsecurePlatformWarning. Both of these are expected;
        #       versions of Python < 2.7.9 have restrictions in their ssl module limiting the configuration
        #       urllib3 can apply. (https://urllib3.readthedocs.org/en/latest/security.html#insecurerequestwarning)
        nginx = load_check('nginx', self.config, self.agent_config)

        # Testing that connection will FAIL with instance 4
        self.assertRaises(requests.exceptions.SSLError, nginx.check, self.config['instances'][4])

    def test_nginx_one_connection(self):
        nginx = load_check('nginx', self.config, self.agent_config)

        # Testing that connection will work with instance 0
        nginx.check(self.config['instances'][0])

        # Checking that only one metric is of type 'nginx.net.connections'
        r = nginx.get_metrics()
        self.assertEquals(len([t for t in r if t[0] == "nginx.net.connections"]), 1, r)

    def test_nginx_plus(self):
        test_data = Fixtures.read_file('nginx_plus_in.json', sdk_dir=FIXTURE_DIR)
        expected = eval(Fixtures.read_file('nginx_plus_out.python', sdk_dir=FIXTURE_DIR))
        nginx = load_check('nginx', self.config, self.agent_config)
        parsed = nginx.parse_json(test_data)
        parsed.sort()

        # Check that the parsed test data is the same as the expected output
        self.assertEquals(parsed, expected)