def test_get_ts_end_date(self):
        v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
        cookies = enhydris_api.login(v['base_url'], v['user'], v['password'])

        # Create a time series in the database
        j = {
            'gentity': v['station_id'],
            'variable': v['variable_id'],
            'unit_of_measurement': v['unit_of_measurement_id'],
            'time_zone': v['time_zone_id'],
        }
        ts_id = enhydris_api.post_model(v['base_url'], cookies, 'Timeseries',
                                        j)

        # Get its last date while it has no data
        date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
        self.assertEqual(date.isoformat(), '0001-01-01T00:00:00')

        # Now upload some data
        ts = Timeseries(ts_id)
        ts.read(StringIO(self.test_timeseries))
        enhydris_api.post_tsdata(v['base_url'], cookies, ts)

        # Get its last date
        date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
        self.assertEqual(date.isoformat(), '2014-01-05T08:00:00')

        # Get the last date of a nonexistent time series
        self.assertRaises(requests.HTTPError, enhydris_api.get_ts_end_date,
                          v['base_url'], cookies, ts_id + 1)

 def get(self, request, pk, format=None):
     ts = Timeseries(id=int(pk))
     self.check_object_permissions(request, ts)
     ts.read_from_db(connection)
     result = StringIO()
     ts.write(result)
     return HttpResponse(result.getvalue(), content_type="text/plain")

 def read_timeseries_from_cache_file(self):
     result = Timeseries()
     if os.path.exists(self.filename):
         with open(self.filename) as f:
             try:
                 result.read_file(f)
             except ValueError:
                 # File may be corrupted; continue with empty time series
                 result = Timeseries()
     return result

 def testUploadTsDataUnauthenticated(self):
     # Attempt to upload some timeseries data, unauthenticated
     response = self.client.put(
         "/api/tsdata/1/",
         encode_multipart(BOUNDARY, {"timeseries_records": "2012-11-06 18:17,20,\n"}),
         content_type=MULTIPART_CONTENT,
     )
     t = Timeseries(1)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 403)
     self.assertEqual(len(t), 0)

 def testUploadTsDataGarbage(self):
     self.assert_(self.client.login(username="user1", password="password1"))
     response = self.client.put(
         "/api/tsdata/1/",
         encode_multipart(BOUNDARY, {"timeseries_records": "2012-aa-06 18:17,20,\n"}),
         content_type=MULTIPART_CONTENT,
     )
     t = Timeseries(1)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 400)
     self.assertEqual(len(t), 0)
     self.client.logout()

 def testUploadTsDataGarbage(self):
     self.assert_(self.client.login(username='user1', password='password1'))
     response = self.client.put(
         "/api/tsdata/{}/".format(self.timeseries1.id),
         encode_multipart(BOUNDARY,
                          {'timeseries_records': '2012-aa-06 18:17,20,\n'}),
         content_type=MULTIPART_CONTENT)
     self.assertEqual(response.status_code, 400)
     t = Timeseries(self.timeseries1.id)
     t.read_from_db(connection)
     self.assertEqual(len(t), 0)
     self.client.logout()

 def check(self, datadir):
     for parm in self.parameters:
         if not parm['ts_id']:
             continue
         actual_ts = Timeseries(parm['ts_id'])
         enhydris_api.read_tsdata(self.base_url, self.cookies, actual_ts)
         reference_ts = Timeseries()
         with open(os.path.join(
                 datadir, 'generated', parm['expname'] + '.txt')) as f:
             reference_ts.read(f)
             precision = self.guess_precision(f)
         self.assertTimeseriesEqual(actual_ts, reference_ts, precision,
                                    parm['expname'] + '.txt')

 def put(self, request, pk, format=None):
     try:
         ts = Timeseries(id=int(pk))
         self.check_object_permissions(request, ts)
         result_if_error = status.HTTP_400_BAD_REQUEST
         ts.read(StringIO(request.DATA['timeseries_records']))
         result_if_error = status.HTTP_409_CONFLICT
         ts.append_to_db(connection, commit=False)
         return HttpResponse(str(len(ts)), content_type="text/plain")
     except ValueError as e:
         return HttpResponse(status=result_if_error,
                             content=str(e),
                             content_type="text/plain")

 def testUploadTsDataAsWrongUser(self):
     # Attempt to upload some timeseries data as user 2; should deny
     self.assert_(self.client.login(username='user2', password='password2'))
     response = self.client.put(
         "/api/tsdata/{}/".format(self.timeseries1.id),
         encode_multipart(BOUNDARY,
                          {'timeseries_records': '2012-11-06 18:17,20,\n'}),
         content_type=MULTIPART_CONTENT)
     t = Timeseries(self.timeseries1.id)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 403)
     self.assertEqual(len(t), 0)
     self.client.logout()

 def testUploadTsDataAsWrongUser(self):
     # Attempt to upload some timeseries data as user 2; should deny
     self.assert_(self.client.login(username="user2", password="password2"))
     response = self.client.put(
         "/api/tsdata/1/",
         encode_multipart(BOUNDARY, {"timeseries_records": "2012-11-06 18:17,20,\n"}),
         content_type=MULTIPART_CONTENT,
     )
     t = Timeseries(1)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 403)
     self.assertEqual(len(t), 0)
     self.client.logout()

 def append_newer_timeseries(self, start_date, ts1):
     self.session_cookies = enhydris_api.login(self.base_url, self.user,
                                               self.password)
     url = self.base_url + 'timeseries/d/{}/download/{}/?version=3'.format(
         self.timeseries_id, start_date.isoformat())
     r = requests.get(url, cookies=self.session_cookies)
     if r.status_code != 200:
         raise HTTPError('Error {} while getting {}'.format(r.status_code,
                                                            url))
     responseio = StringIO(r.text)
     ts2 = Timeseries()
     ts2.read_file(responseio)
     responseio.seek(0)
     ts1.read_meta(responseio)
     ts1.append(ts2)

 def time_step(self):
     """
     Return time step of all time series. If time step is not the same
     for all time series, raises exception.
     """
     time_step = None
     for filename in self.files:
         with open(filename) as f:
             t = Timeseries()
             t.read_meta(f)
         item_time_step = (t.time_step.length_minutes,
                           t.time_step.length_months)
         if time_step and (item_time_step != time_step):
             raise WrongValueError(
                 'Not all time series have the same step')
         time_step = item_time_step
     return time_step

 def setUp(self):
     get_server_from_env(self.__dict__)
     self.ref_ts = Timeseries(0)
     if not self.base_url:
         return
     self.cookies = enhydris_api.login(self.base_url, self.user,
                                       self.password)
     self.timeseries_id = create_timeseries(self.cookies, self.__dict__)
     self.ts = Timeseries(self.timeseries_id)

def h_integrate(mask, stations_layer, date, output_filename_prefix, date_fmt,
                funct, kwargs):
    date_fmt_for_filename = date.strftime(date_fmt).replace(' ', '-').replace(
        ':', '-')
    output_filename = '{}-{}.tif'.format(output_filename_prefix,
                                         date.strftime(date_fmt_for_filename))
    if not _needs_calculation(output_filename, date, stations_layer):
        return

    # Read the time series values and add the 'value' attribute to
    # stations_layer
    stations_layer.CreateField(ogr.FieldDefn('value', ogr.OFTReal))
    input_files = []
    stations_layer.ResetReading()
    for station in stations_layer:
        filename = station.GetField('filename')
        t = Timeseries()
        with open(filename) as f:
            t.read_file(f)
        value = t.get(date, float('NaN'))
        station.SetField('value', value)
        if not isnan(value):
            input_files.append(filename)
        stations_layer.SetFeature(station)
    if not input_files:
        return

    # Create destination data source
    output = gdal.GetDriverByName('GTiff').Create(
        output_filename, mask.RasterXSize, mask.RasterYSize, 1,
        gdal.GDT_Float32)
    output.SetMetadataItem('TIMESTAMP', date.strftime(date_fmt))
    output.SetMetadataItem('INPUT_FILES', '\n'.join(input_files))

    try:
        # Set geotransform and projection in the output data source
        output.SetGeoTransform(mask.GetGeoTransform())
        output.SetProjection(mask.GetProjection())

        # Do the integration
        integrate(mask, stations_layer, output.GetRasterBand(1), funct, kwargs)
    finally:
        # Close the dataset
        output = None

def create_objects(dma, household_identifier, series, force=False):
    """
    When a household is fully parsed then this command is called to create
    database objects thus: user (household owner), household, database time
    series placeholders (for raw data and for processed data), to write actual
    time series data in database and finally to estimate the household
    occupancy.
    """
    print "Processing household %s, user username will be %s as well"%(
            household_identifier, household_identifier)
    # Create user (household owner), household, database series placeholders
    user = create_user(household_identifier)
    household=create_household(household_identifier, user,
            zone=dma.id)
    db_series = create_raw_timeseries(household)
    create_processed_timeseries(household)
    timeseries_data = {}
    # Now we will create timeseries.Timeseries() and we will add
    # parsed values
    for variable in db_series:
        if variable not in ('WaterCold', 'Electricity'):
            continue
        s, e = timeseries_bounding_dates_from_db(db.connection,
                db_series[variable].id)
        if not force and (s or e):
            print 'Raw timeseries id=%s has already data, skipping...'%(
                    db_series[variable].id,)
            continue
        timeseries = TSeries()
        timeseries.id = db_series[variable].id
        total = 0.0
        for timestamp, value in series[variable]:
            if not math.isnan(value):
                total += value
                timeseries[timestamp] = total
            else:
                timeseries[timestamp] = float('NaN')
        timeseries_data[variable] = timeseries
        timeseries.write_to_db(db=db.connection,
                transaction=transaction,
                commit=False)
    if 'WaterCold' in timeseries_data:
        calc_occupancy(timeseries_data['WaterCold'], household)

def _needs_calculation(output_filename, date, stations_layer):
    """
    Used by h_integrate to check whether the output file needs to be calculated
    or not. It does not need to be calculated if it already exists and has been
    calculated from all available data.
    """
    # Return immediately if output file does not exist
    if not os.path.exists(output_filename):
        return True

    # Get list of files which were used to calculate the output file
    fp = gdal.Open(output_filename)
    try:
        actual_input_files = fp.GetMetadataItem('INPUT_FILES')
        if actual_input_files is None:
            raise IOError('{} does not contain the metadata item INPUT_FILES'
                          .format(output_filename))
    finally:
        fp = None  # Close file
    actual_input_files = set(actual_input_files.split('\n'))

    # Get list of files available for calculating the output file
    stations_layer.ResetReading()
    available_input_files = set(
        [station.GetField('filename') for station in stations_layer
         if os.path.exists(station.GetField('filename'))])

    # Which of these files have not been used?
    unused_files = available_input_files - actual_input_files

    # For each one of these files, check whether it has newly available data.
    # Upon finding one that does, the verdict is made: return True
    for filename in unused_files:
        t = Timeseries()
        with open(filename) as f:
            t.read_file(f)
        value = t.get(date, float('NaN'))
        if not isnan(value):
            return True

    # We were unable to find data that had not already been used
    return False

    def test_execute(self):
        application = AggregateApp()
        application.read_command_line()
        application.read_configuration()

        # Verify the output files don't exist yet
        self.assertFalse(os.path.exists(self.output_filenames[0]))
        self.assertFalse(os.path.exists(self.output_filenames[1]))

        # Execute
        application.run()

        # Check that it has created two files
        self.assertTrue(os.path.exists(self.output_filenames[0]))
        self.assertTrue(os.path.exists(self.output_filenames[1]))

        # Check that the created time series are correct
        t = Timeseries()
        with open(self.output_filenames[0]) as f:
            t.read_file(f)
        self.assertEqual(t.timezone, 'EET (UTC+0200)')
        self.assertEqual(len(t), 1)
        self.assertAlmostEqual(t['2014-06-16 16:00'], 114.9, places=5)
        t = Timeseries()
        with open(self.output_filenames[1]) as f:
            t.read_file(f)
        self.assertEqual(t.timezone, '')
        self.assertEqual(len(t), 1)
        self.assertAlmostEqual(t['2014-06-17 16:00'], 50.8167, places=5)

    def setUp(self):
        self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
        self.cookies = enhydris_api.login(self.parms['base_url'],
                                          self.parms['user'],
                                          self.parms['password'])

        # Create two time series
        j = {
            'gentity': self.parms['station_id'],
            'variable': self.parms['variable_id'],
            'unit_of_measurement': self.parms['unit_of_measurement_id'],
            'time_zone': self.parms['time_zone_id'],
            'time_step': 3,
            'timestamp_offset_minutes': 0,
            'timestamp_offset_months': 0,
            'remarks': 'Très importante',
        }
        self.ts1_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Timeseries', j)
        self.ts2_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Timeseries', j)
        assert self.ts1_id != self.ts2_id

        # Add some data (all but the last record) to the database
        ts = Timeseries(self.ts1_id)
        ts.read(StringIO(self.timeseries1_top))
        enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
        ts = Timeseries(self.ts2_id)
        ts.read(StringIO(self.timeseries2_top))
        enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)

        # Temporary directory for cache files
        self.tempdir = tempfile.mkdtemp()
        self.savedcwd = os.getcwd()
        os.chdir(self.tempdir)

    def setUp(self):
        parms = settings.BITIA_TEST_ENHYDRIS_INSTALLATION
        self.cookies = enhydris_api.login(parms['base_url'],
                                          parms['user'],
                                          parms['password'])

        # Create two time series
        j = {
            'gentity': parms['station_id'],
            'variable': parms['variable_id'],
            'unit_of_measurement': parms['unit_of_measurement_id'],
            'time_zone': parms['time_zone_id'],
        }
        self.ts1_id = enhydris_api.post_model(
            parms['base_url'], self.cookies, 'Timeseries', j)
        self.ts2_id = enhydris_api.post_model(
            parms['base_url'], self.cookies, 'Timeseries', j)
        assert self.ts1_id != self.ts2_id

        # Add some data (all but the last record) to the database
        ts = Timeseries(self.ts1_id)
        ts.read(StringIO(self.timeseries1_top))
        enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)
        ts = Timeseries(self.ts2_id)
        ts.read(StringIO(self.timeseries2_top))
        enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)

        # Temporary directory for cache files
        self.tempdir = tempfile.mkdtemp()

def create_ogr_layer_from_timeseries(filenames, epsg, data_source):
    # Prepare the co-ordinate transformation from WGS84 to epsg
    source_sr = osr.SpatialReference()
    source_sr.ImportFromEPSG(4326)
    target_sr = osr.SpatialReference()
    target_sr.ImportFromEPSG(epsg)
    transform = osr.CoordinateTransformation(source_sr, target_sr)

    layer = data_source.CreateLayer('stations', target_sr)
    layer.CreateField(ogr.FieldDefn('filename', ogr.OFTString))
    for filename in filenames:
        with open(filename) as f:
            ts = Timeseries()
            ts.read_meta(f)
        point = ogr.Geometry(ogr.wkbPoint)
        point.AddPoint(ts.location['abscissa'], ts.location['ordinate'])
        point.Transform(transform)
        f = ogr.Feature(layer.GetLayerDefn())
        f.SetGeometry(point)
        f.SetField('filename', filename)
        layer.CreateFeature(f)
    return layer