def test_seed_refresh_remove_before_from_file(self):
        # tile already there but too old, will be refreshed and removed
        t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
        with tmp_image((256, 256), format='png') as img:
            img_data = img.read()
            expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                                  '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                                  '&width=256&height=128&srs=EPSG:4326'},
                            {'body': img_data, 'headers': {'content-type': 'image/png'}})
            with mock_httpd(('localhost', 42423), [expected_req]):
                # touch the seed_conf file and refresh everything
                timestamp = time.time() - 60
                os.utime(self.seed_conf_file, (timestamp, timestamp))
                
                with local_base_config(self.mapproxy_conf.base_config):
                    seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                    tasks = seed_conf.seeds(['refresh_from_file'])

                    seed(tasks, dry_run=False)

                assert os.path.exists(t000)
                assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
        
                # now touch the seed_conf again and remove everything
                os.utime(self.seed_conf_file, None)
                with local_base_config(self.mapproxy_conf.base_config):
                    seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                    cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
                    cleanup(cleanup_tasks, verbose=False, dry_run=False)

 def test_reseed_uptodate(self):
     # tile already there.
     self.make_tile((0, 0, 0))
     seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
     tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
     seed(tasks, dry_run=False)
     cleanup(cleanup_tasks, verbose=False, dry_run=False)

    def test_timeout(self):
        # test concurrent seeding where seed concurrency is higher than the permitted
        # concurrent_request value of the source and a lock times out

        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        tasks = seed_conf.seeds(['test'])
        
        expected_req1 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                          '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                          '&width=256&height=128&srs=EPSG:4326'},
                        {'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})
        
        expected_req2 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                          '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                          '&width=512&height=256&srs=EPSG:4326'},
                        {'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})

        expected_req3 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                          '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                          '&width=1024&height=512&srs=EPSG:4326'},
                        {'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})


        with mock_httpd(('localhost', 42423), [expected_req1, expected_req2, expected_req3], unordered=True):
            seed(tasks, dry_run=False, concurrency=3)

    def test_cleanup_remove_all(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['remove_all'])

        self.make_tile((0, 0, 0))
        self.make_tile((0, 0, 1))
        self.make_tile((1, 0, 1))
        self.make_tile((0, 1, 1))
        self.make_tile((1, 1, 1))
        self.make_tile((0, 0, 2))
        self.make_tile((0, 0, 3))

        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
            ['00', '01', '02', '03'])

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert self.tile_exists((0, 0, 0))
        assert not self.tile_exists((0, 0, 1))
        assert not self.tile_exists((1, 0, 1))
        assert not self.tile_exists((0, 1, 1))
        assert not self.tile_exists((1, 1, 1))
        assert not self.tile_exists((0, 0, 1))
        assert self.tile_exists((0, 0, 2))
        assert self.tile_exists((0, 0, 3))

        # remove_all should remove the whole directory
        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
            ['00', '02', '03'])

    def test_cleanup_mbtiles(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])
        
        cache = tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile())

        cleanup(cleanup_tasks, verbose=False, dry_run=False)

 def test_seed(self):
     with tmp_image((256, 256), format='png') as img:
         img_data = img.read()
         expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                               '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                               '&width=256&height=128&srs=EPSG:4326'},
                         {'body': img_data, 'headers': {'content-type': 'image/png'}})
         with mock_httpd(('localhost', 42423), [expected_req]):
             seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
             tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
             seed(tasks, dry_run=False)
             cleanup(cleanup_tasks, verbose=False, dry_run=False)

    def test_reseed_mbtiles_with_refresh(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])
        
        cache = tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile())

        expected_req = ({'path': r'/service?LAYERS=bar&SERVICE=WMS&FORMAT=image%2Fpng'
                          '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                          '&width=256&height=128&srs=EPSG:4326'},
                        {'body': tile_image, 'headers': {'content-type': 'image/png'}})
        with mock_httpd(('localhost', 42423), [expected_req]):
            # mbtiles does not support timestamps, refresh all tiles
            seed(tasks, dry_run=False)

 def test_cleanup_levels(self):
     seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
     cleanup_tasks = seed_conf.cleanups(['cleanup'])
     
     self.make_tile((0, 0, 0))
     self.make_tile((0, 0, 1))
     self.make_tile((0, 0, 2))
     self.make_tile((0, 0, 3))
     
     cleanup(cleanup_tasks, verbose=False, dry_run=False)
     assert not self.tile_exists((0, 0, 0))
     assert not self.tile_exists((0, 0, 1))
     assert self.tile_exists((0, 0, 2))
     assert not self.tile_exists((0, 0, 3))

    def test_cleanup_levels(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['cleanup'])

        self.make_tile((0, 0, 0))
        self.make_tile((0, 0, 1))
        self.make_tile((0, 0, 2))
        self.make_tile((0, 0, 3))

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert not self.tile_exists((0, 0, 0))
        assert not self.tile_exists((0, 0, 1))
        assert self.tile_exists((0, 0, 2))
        assert not self.tile_exists((0, 0, 3))

        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
            ['02'])

    def test_cleanup_sqlite_remove_all(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['sqlite_cache_remove_all'])

        cache = cleanup_tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile((0, 0, 2)))
        cache.store_tile(self.create_tile((0, 0, 3)))
        assert cache.is_cached(Tile((0, 0, 2)))
        assert cache.is_cached(Tile((0, 0, 3)))

        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))),
            ['2.mbtile', '3.mbtile'])

        cleanup(cleanup_tasks, verbose=False, dry_run=False)

        # 3.mbtile file should be removed completely
        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))),
            ['3.mbtile'])
        assert not cache.is_cached(Tile((0, 0, 2)))
        assert cache.is_cached(Tile((0, 0, 3)))

 def test_reseed_remove_before(self):
     # tile already there but too old
     t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
     # old tile outside the seed view (should be removed)
     t001 = self.make_tile((0, 0, 1), timestamp=time.time() - (60*60*25))
     assert os.path.exists(t000)
     assert os.path.exists(t001)
     with tmp_image((256, 256), format='png') as img:
         img_data = img.read()
         expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                               '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                               '&width=256&height=128&srs=EPSG:4326'},
                         {'body': img_data, 'headers': {'content-type': 'image/png'}})
         with mock_httpd(('localhost', 42423), [expected_req]):
             seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
             tasks, cleanup_tasks = seed_conf.seeds(), seed_conf.cleanups()
             seed(tasks, dry_run=False)
             cleanup(cleanup_tasks, verbose=False, dry_run=False)
     
     assert os.path.exists(t000)
     assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
     assert not os.path.exists(t001)

    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error('missing seed_conf file as last argument or --seed-conf option')
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " + '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            progress = None
            if options.continue_seed or options.progress_file:
                if options.progress_file:
                    progress_file = options.progress_file
                else:
                    progress_file = '.mapproxy_seed_progress'
                progress = ProgressStore(progress_file,
                    continue_seed=options.continue_seed)

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print('Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
                        progress_store=progress)
                    seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
                         concurrency=options.concurrency, cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' % (
                        len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
                    cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
                            concurrency=options.concurrency, progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()

 def test_seed_dry_run(self):
     seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
     tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
     seed(tasks, dry_run=True)
     cleanup(cleanup_tasks, verbose=False, dry_run=True)

 def test_active_seed_tasks(self):
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         assert len(seed_conf.seed_tasks_names()) == 5
         assert len(seed_conf.seeds()) == 5

def grids_command(args=None):
    parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
        help="MapProxy configuration.")
    parser.add_option("-g", "--grid", dest="grid_name",
        help="Display only information about the specified grid.")
    parser.add_option("--all", dest="show_all", action="store_true", default=False,
        help="Show also grids that are not referenced by any cache.")
    parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache")
    coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage")
    coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined")
    coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given")

    from mapproxy.script.util import setup_logging
    import logging
    setup_logging(logging.WARN)

    if args:
        args = args[1:] # remove script name

    (options, args) = parser.parse_args(args)
    if not options.mapproxy_conf:
        if len(args) != 1:
            parser.print_help()
            sys.exit(1)
        else:
            options.mapproxy_conf = args[0]
    try:
        proxy_configuration = load_configuration(options.mapproxy_conf)
    except IOError as e:
        print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
        sys.exit(2)
    except ConfigurationError as e:
        print(e, file=sys.stderr)
        print('ERROR: invalid configuration (see above)', file=sys.stderr)
        sys.exit(2)

    with local_base_config(proxy_configuration.base_config):
        if options.show_all or options.grid_name:
            grids = proxy_configuration.grids
        else:
            caches = proxy_configuration.caches
            grids = {}
            for cache in caches.values():
                grids.update(cache.grid_confs())
            grids = dict(grids)

        if options.grid_name:
            options.grid_name = options.grid_name.lower()
            # ignore case for keys
            grids = dict((key.lower(), value) for (key, value) in iteritems(grids))
            if not grids.get(options.grid_name, False):
                print('grid not found: %s' % (options.grid_name,))
                sys.exit(1)

        coverage = None
        if options.coverage and options.seed_config:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
            except SeedConfigurationError as e:
                print('ERROR: invalid configuration (see above)', file=sys.stderr)
                sys.exit(2)

            if not isinstance(seed_conf, SeedingConfiguration):
                print('Old seed configuration format not supported')
                sys.exit(1)

            coverage = seed_conf.coverage(options.coverage)
            coverage.name = options.coverage

        elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
            print('--coverage and --seed-conf can only be used together')
            sys.exit(1)

        if options.list_grids:
            display_grids_list(grids)
        elif options.grid_name:
            display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage)
        else:
            display_grids(grids, coverage=coverage)

            grids.update(cache.grid_confs())
        grids = dict(grids)

    if options.grid_name:
        options.grid_name = options.grid_name.lower()
        # ignore case for keys
        grids = dict((key.lower(), value) for (key, value) in grids.iteritems())
        if not grids.get(options.grid_name, False):
            print 'grid not found: %s' % (options.grid_name,)
            sys.exit(1)

    coverage = None
    if options.coverage and options.seed_config:
        with local_base_config(proxy_configuration.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
            except SeedConfigurationError, e:
                print >>sys.stderr, 'ERROR: invalid configuration (see above)'
                sys.exit(2)

            if not isinstance(seed_conf, SeedingConfiguration):
                print 'Old seed configuration format not supported'
                sys.exit(1)

            coverage = seed_conf.coverage(options.coverage)
            coverage.name = options.coverage

    elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
        print '--coverage and --seed-conf can only be used together'
        sys.exit(1)

 def test_active_seed_tasks(self):
     seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
     assert len(seed_conf.seed_tasks_names()) == 4
     assert len(seed_conf.seeds()) == 3