def test_site_data_update_site_list_with_multiple_updates(test_file_path):
    first_timestamp = '2013-01-01T01:01:01'
    second_timestamp = '2013-02-02T02:02:02'
    site_code = '01117800'
    site_data_file = test_util.get_test_file_path(
        'usgs/nwis/site_%s_daily.xml' % site_code)
    with test_util.mocked_urls(site_data_file):
        with freezegun.freeze_time(first_timestamp):
            nwis.hdf5.update_site_data(site_code, path=test_file_path,
                    autorepack=False)
    site_data = nwis.hdf5.get_site_data(site_code, path=test_file_path)

    last_value = site_data['00060:00003']['values'][-1]

    assert first_timestamp == last_value['last_checked'] == last_value['last_modified']

    update_data_file = test_util.get_test_file_path(
        'usgs/nwis/site_%s_daily_update.xml' % site_code)
    with test_util.mocked_urls(update_data_file):
        with freezegun.freeze_time(second_timestamp):
            nwis.hdf5.update_site_data(site_code, path=test_file_path,
                    autorepack=False)
    updated_site_data = nwis.hdf5.get_site_data(site_code, path=test_file_path)

    updated_values = updated_site_data['00060:00003']['values']
    last_value = updated_values[-1]
    assert last_value['last_checked'] != first_timestamp
    assert second_timestamp == last_value['last_checked'] == last_value['last_modified']

    original_timestamp = first_timestamp
    modified_timestamp = second_timestamp

    test_values = [
        dict(datetime="1963-01-23T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='7'),
        dict(datetime="1964-01-23T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='1017'),
        dict(datetime="1964-01-24T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="A", value='191'),
        dict(datetime="1964-08-22T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="A", value='7.9'),
        dict(datetime="1969-05-26T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='1080'),
        dict(datetime="2011-12-06T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='222'),
        dict(datetime="2011-12-15T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="P Eqp", value='-999999'),
        dict(datetime="2012-01-15T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="P e", value='97'),
        dict(datetime="2012-05-25T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='56'),
        dict(datetime="2012-05-26T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='55'),
        dict(datetime="2012-05-27T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='52'),
        dict(datetime="2012-05-28T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='48'),
        dict(datetime="2012-05-29T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1099'),
        dict(datetime="2012-05-30T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1098'),
        dict(datetime="2012-05-31T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='41'),
        dict(datetime="2012-06-01T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='37'),
        dict(datetime="2012-06-02T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1097'),
        dict(datetime="2012-06-03T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='69'),
        dict(datetime="2012-06-04T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='81'),
        dict(datetime="2012-06-05T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1071'),
        dict(datetime="2012-06-06T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='2071'),
    ]

    for test_value in test_values:
        assert updated_values.index(test_value) >= 0

def test_get_station_data():
    test_station_data = [
        ('MYST2', '2018-02-03', {
            'code': 'MYST2',
            'description': 'Pat Mayse Lake',
            'station_type': 'RESERVOIR',
            'timezone': 'US/Central',
            'values': {
                '2018-02-03 01:00:00': {'PRECIP PRE': 0.0, 'CIP(A) ELE': 0.0, 'VATION   S': 451.61, 'TORAGE': 121347.0, 'INFLOW   R': 0.0, 'ELEASE  AI': 59.0, 'R-TEMP  WI': 35.4, 'ND-DIRWIND': 75.0, '-SPEED REL': 5.36, '-HUMID SOL': 41.52, 'AR-RAD   V': -2.0, 'OLTAGE  BA': 12.37}
                },
        }),
    ]

    for code, date, test_data in test_station_data:
        url_date = date.replace('-', '')
        filename = '%s.%s.html' % (code, url_date)
        data_file = 'usace/swtwc/' + filename
        with test_util.mocked_urls(data_file):
            station_data = ulmo.usace.swtwc.get_station_data(code, date)

        for key, value in test_data.items():
            if key == 'values':
                _compare_values(test_data['values'], station_data['values'])
            else:
                assert station_data[key] == test_data[key]

def test_multi_message_download():
    for test_set in multi_message_test_sets:
        with test_util.mocked_urls(test_set["data_files"]):
            data = ulmo.usgs.eddn.get_data(test_set["dcp_address"], start=test_set["start"])
            assert data["message_timestamp_utc"][-1] == test_set["first_row_message_timestamp_utc"]
            assert data["message_timestamp_utc"][0] == test_set["last_row_message_timestamp_utc"]
            assert len(data) == test_set["number_of_lines"]

def test_get_sites_by_multiple_serving_parameter_code():
    site_code = '08068500'
    parameter_code_values = '00060,00065'
    sites_data_file = 'usgs/nwis/sites_%s_%s_daily.xml' % (site_code, parameter_code_values)
    with test_util.mocked_urls(sites_data_file):
        sites = ulmo.usgs.nwis.get_sites(sites=site_code, parameter_code=parameter_code_values, service='dv')
    assert len(sites) == 1

def test_last_refresh_gets_updated(test_file_path):
    test_file_path = test_file_path + "test.h5"

    first_timestamp = '2013-01-01T01:01:01'
    second_timestamp = '2013-02-02T02:02:02'
    forth_timestamp = '2013-03-03T03:03:03'
    site_code = '01117800'
    site_data_file = test_util.get_test_file_path(
        'usgs/nwis/site_%s_daily.xml' % site_code)

    with test_util.mocked_urls(site_data_file):
        with freezegun.freeze_time(first_timestamp):
            nwis.hdf5.update_site_data(site_code, path=test_file_path,
                    autorepack=False)
        first_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
        assert first_refresh == first_timestamp

        with freezegun.freeze_time(second_timestamp):
            nwis.hdf5.update_site_data(site_code, path=test_file_path,
                    autorepack=False)
        second_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
        assert second_refresh == second_timestamp

        nwis.hdf5.update_site_data(site_code, path=test_file_path,
                input_file=site_data_file, autorepack=False)
        third_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
        assert third_refresh == None

        with freezegun.freeze_time(forth_timestamp):
            nwis.hdf5.update_site_data(site_code, path=test_file_path,
                    autorepack=False)
        forth_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
        assert forth_refresh is not None
        assert forth_refresh == forth_timestamp

def test_get_sensors():
    sensors_file = 'cdec/historical/sensors.htm'
    with test_util.mocked_urls(sensors_file):
        sensors = ulmo.cdec.historical.get_sensors()
    assert 200 < len(sensors)
    sensors.index = sensors['Sensor No'].astype(int)
    assert u'FLOW, RIVER DISCHARGE' == sensors.xs(20)['Description']

def test_get_site_data_bad_service_raises_error():
    site_code = '08068500'
    site_data_file = 'usgs/nwis/site_%s_daily.xml' % site_code
    with test_util.mocked_urls(site_data_file):
        with pytest.raises(ValueError):
            ulmo.usgs.nwis.get_site_data(site_code,
                    service="bad_service")

def test_get_stations_with_elements():
    test_elements = [
        {
            'elements': 'PRCP',
            'includes': [
                'ASN00008230',
                'WA006567710',
                'VQC00672823',
            ],
            'excludes': ['SWE00136141', 'USR0000OHOR', 'USC00450935', 'USR0000THEN', 'USR0000MDRY'],
        }, {
            'elements': ['SNOW', 'TMAX'],
            'includes': [
                'ACW00011604',
                'USW00094895',
                'VQW00011640',
                'ZI000067991',
            ],
            'excludes': ['BR037642870', 'BR00C8-0100', 'BR048519530', 'BR002548070', 'IN009081600']
        },
    ]
    url_files = {
        'ghcnd-stations': 'ncdc/ghcnd/ghcnd-stations.txt',
        'ghcnd-inventory': 'ncdc/ghcnd/ghcnd-inventory.txt'
    }

    with test_util.mocked_urls(url_files):
        for test_element in test_elements:
            elements = test_element.get('elements')
            stations = ghcn_daily.get_stations(elements=elements,
                as_dataframe=True)
            _check_stations_dataframe(stations,
                test_element.get('includes'),
                test_element.get('excludes'))

def test_get_site_data_multiple_methods():
    site_code = '08054500'
    site_data_file = 'usgs/nwis/site_08054500_multiple_methods.xml'
    with test_util.mocked_urls(site_data_file):
        site_data = ulmo.usgs.nwis.get_site_data(site_code, methods={'62614': 'all', '45592': 'all'})
        assert len(site_data['00054:00003']['values']) == 1
        assert len(site_data.keys()) == 17

def test_get_stations_with_elements():
    test_elements = [
        {
            'elements': 'PRCP',
            'includes': [
                'ASN00008230',
                'WA006567710',
                'VQC00672823',
            ],
            'excludes': [
                'AR000870470',
                'AR000875850',
                'BC000068234',
                'GME00111464',
                'UY000864400',
            ],
        }, {
            'elements': 'PRCP',
            'includes': [
                'ASN00008230',
                'WA006567710',
                'VQC00672823',
            ],
            'excludes': [
                'AR000870470',
                'AR000875850',
                'BC000068234',
                'GME00111464',
                'UY000864400',
            ],
        }, {
            'elements': ['SNOW', 'TMAX'],
            'includes': [
                'ACW00011604',
                'USW00094895',
                'VQW00011640',
                'ZI000067991',
            ],
            'excludes': [
                'BR00B4-0010',
                'IN003070101',
                'KZ000038223',
                'ZA000067753',
            ],
        },
    ]
    url_files = {
        'ghcnd-stations': 'ncdc/ghcnd/ghcnd-stations.txt',
        'ghcnd-inventory': 'ncdc/ghcnd/ghcnd-inventory.txt'
    }

    with test_util.mocked_urls(url_files):
        for test_element in test_elements:
            elements = test_element.get('elements')
            stations = ghcn_daily.get_stations(elements=elements,
                as_dataframe=True)
            _check_stations_dataframe(stations,
                test_element.get('includes'),
                test_element.get('excludes'))

def test_get_stations_as_dicts():
    with test_util.mocked_urls('ncdc/ghcnd/ghcnd-stations.txt'):
        stations = ghcn_daily.get_stations()
    assert len(stations) > 80000

    for test_station in test_stations:
        station_id = test_station.get('id')
        assert stations.get(station_id) == test_station

def test_get_station_data_out_of_range():
    # can't easily test current since it is a moving target changes, but mostly
    # just make sure it parses correctl: current will have '---' values where
    # previous days do not
    data_file = "usace/swtwc/empty.html"
    with test_util.mocked_urls(data_file):
        with pytest.raises(ValueError):
            station_data = ulmo.usace.swtwc.get_station_data("MYST2", "1945-01-01")

def test_get_station_data_current():
    # can't easily test current since it is a moving target changes, but mostly
    # just make sure it parses correctl: current will have '---' values where
    # previous days do not
    data_file = "usace/swtwc/MYST2.current.html"
    with test_util.mocked_urls(data_file):
        station_data = ulmo.usace.swtwc.get_station_data("MYST2")
    assert len(station_data.get("values")) > 0

def test_get_site_data_single_site_with_start_and_end():
    site_code = '08068500'
    site_data_file = 'usgs/nwis/site_08068500_instantaneous_2011-11-05_2011-11-18.xml'
    with test_util.mocked_urls(site_data_file):
        site_data = ulmo.usgs.nwis.get_site_data(site_code, start='2011-11-05',
                end='2011-11-18', service='instantaneous')
    assert len(site_data) == 7
    assert len(site_data['63680:00011']['values']) == 1250

def test_get_site_data_multiple_methods():
    site_code = '08054500'
    site_data_file = 'usgs/nwis/site_08054500_multiple_methods.xml'
    with test_util.mocked_urls(site_data_file):
        site_data = ulmo.usgs.nwis.get_site_data(site_code, methods={'00062': 'all'})
        assert len(site_data['00062:00011:1']['values']) == 288
        assert len(site_data['00062:00011:20']['values']) == 288
        assert len(site_data.keys()) == 2

def test_get_data_by_state():
    for test_set in test_sets:
        with test_util.mocked_urls(test_set['filename']):
            data = ulmo.cpc.drought.get_data(
                state=test_set['state'], start=test_set['start'],
                end=test_set['end'])
        assert len(data) == 1
        assert test_set['state'] in data

def test_get_site_data_single_site_with_period():
    site_data_file = 'usgs/nwis/site_01117800_instantaneous_P45D.xml'
    site_code = '01117800'
    with test_util.mocked_urls(site_data_file):
        site_data = ulmo.usgs.nwis.get_site_data(site_code, period='P45D',
                service='daily')
    assert len(site_data) >= 1
    assert len(site_data['00060:00003']['values']) == 45

def test_get_data():
    for test_set in test_sets:
        with test_util.mocked_urls(test_set['filename']):
            data = ulmo.cpc.drought.get_data(start=test_set['start'], end=test_set['end'])

        values = data.get(test_set['state'], {}).get(test_set['climate_division'])
        test_values = test_set['values']

        for test_value in test_values:
            assert test_value in values

def test_get_stations_as_dataframe():
    with test_util.mocked_urls('ncdc/ghcnd/ghcnd-stations.txt'):
        stations = ghcn_daily.get_stations(as_dataframe=True)
    assert len(stations) > 80000

    for test_station in test_stations:
        station_id = test_station.get('id')
        station = stations.xs(station_id)
        station_dict = util.misc._nans_to_nones(station.to_dict())
        assert station_dict == test_station

def test_get_historical_data():
    parameters_file = 'lcra/waterquality/12147_params.html'
    data_file = 'lcra/waterquality/12147_results.html'

    with test_util.mocked_urls(data_file):
        results = get_historical_data(12147)

    assert len(results) == 12
    for data in results:
        assert data['Site'] == u'12147'