def _is_valid_models_list_for_model(models, model_name, ModelClass, instance):
    """Validates everything in models are valid filters for model_name"""
    def collection_udf_set_for_model(Model):
        if not issubclass(ModelClass, UDFModel):
            return {}
        if hasattr(Model, 'instance'):
            fake_model = Model(instance=instance)
        else:
            fake_model = Model()
        return set(fake_model.collection_udfs_search_names())

    # MapFeature is valid for all models
    models = models - MAP_FEATURE_RELATED_NAMES

    object_name = to_object_name(model_name)
    models = models - {object_name}

    if model_name == 'Plot':
        related_models = PLOT_RELATED_MODELS
    else:
        related_models = {ModelClass}

    for Model in related_models:
        models = models - {to_object_name(Model.__name__)}
        if issubclass(Model, UDFModel):
            models = models - collection_udf_set_for_model(Model)

    return len(models) == 0

def add_udfs_to_web_detail_fields(apps, schema_editor):
    Instance = apps.get_model("treemap", "Instance")
    UserDefinedFieldDefinition = apps.get_model("treemap", "UserDefinedFieldDefinition")

    for instance in Instance.objects.all():
        scalar_udfs = UserDefinedFieldDefinition.objects \
            .filter(iscollection=False, instance=instance) \
            .order_by('name')

        # Nothing to add to the configs
        if not scalar_udfs:
            continue

        if 'web_detail_fields' not in instance.config:
            instance.config['web_detail_fields'] = deepcopy(DEFAULT_WEB_DETAIL_FIELDS)

        for udf in scalar_udfs:
            for group in instance.config['web_detail_fields']:
                if 'model' in group and group['model'] == to_object_name(udf.model_type):
                    field_keys = group.get('field_keys')

                    udf_full_name = to_object_name(udf.model_type) + '.udf:' + udf.name
                    if 'field_keys' in group and udf_full_name not in field_keys:
                        field_keys.append(udf_full_name)

        instance.save()

    def get_udfc_search_fields(self, user):
        from treemap.models import InstanceUser
        from treemap.udf import UDFModel
        from treemap.util import to_object_name, leaf_models_of_class
        from treemap.lib.perms import udf_write_level, READ, WRITE

        try:
            iu = self.instanceuser_set.get(user__pk=user.pk)
        except InstanceUser.DoesNotExist:
            iu = None

        data = DotDict({"models": set(), "udfc": {}})
        for clz in leaf_models_of_class(UDFModel):
            model_name = clz.__name__
            for k, v in clz.collection_udf_settings.items():
                udfds = (u for u in udf_defs(self, model_name) if u.name == k)
                for udfd in udfds:
                    if udf_write_level(iu, udfd) in (READ, WRITE):
                        _base_nest_path = "udfc.%s." % (to_object_name(k))
                        ids_nest_path = "%sids.%s" % (_base_nest_path, to_object_name(model_name))
                        models_nest_path = "%smodels.%s" % (_base_nest_path, to_object_name(model_name))
                        data[ids_nest_path] = udfd.pk
                        data[models_nest_path] = {"udfd": udfd, "fields": udfd.datatype_dict[0]["choices"]}
                        p = "udfc.%s." % to_object_name(k)
                        data[p + "action_verb"] = v["action_verb"]
                        data[p + "range_field_key"] = v["range_field_key"]
                        data[p + "action_field_key"] = v["action_field_key"]
                        data["models"] |= {clz}

        return data

    def get_udfc_search_fields(self, user):
        from treemap.models import InstanceUser
        from treemap.udf import UDFModel
        from treemap.util import to_object_name, leaf_models_of_class
        from treemap.lib.perms import udf_write_level, READ, WRITE

        try:
            iu = self.instanceuser_set.get(user__pk=user.pk)
        except InstanceUser.DoesNotExist:
            iu = None

        data = DotDict({'models': set(), 'udfc': {}})
        for clz in (leaf_models_of_class(UDFModel)):
            model_name = clz.__name__
            items = (
                (k, v) for k, v
                in getattr(clz, 'collection_udf_settings', {}).iteritems())
            for k, v in items:
                udfds = (u for u in udf_defs(self, model_name) if u.name == k)
                for udfd in udfds:
                    if udf_write_level(iu, udfd) in (READ, WRITE):
                        nest_path = ('udfc.%s.models.%s' %
                                     (to_object_name(k),
                                      to_object_name(model_name)))
                        data[nest_path] = {
                            'udfd': udfd,
                            'fields': udfd.datatype_dict[0]['choices']
                        }
                        p = 'udfc.%s.' % to_object_name(k)
                        data[p + 'action_verb'] = v['action_verb']
                        data[p + 'range_field_key'] = v['range_field_key']
                        data[p + 'action_field_key'] = v['action_field_key']
                        data['models'] |= {clz}

        return data

    def _get_udfc_search_fields(self):
        from treemap.util import to_object_name

        empty_udfc = {to_object_name(n_k):
                      {to_object_name(m_k): {'fields': [], 'udfd': None}
                       for m_k in UDFC_MODELS}
                      for n_k in UDFC_NAMES}

        udfds = []
        for model_name in UDFC_MODELS:
            for udfd in udf_defs(self, model_name):
                if udfd.name in UDFC_NAMES:
                    udfds.append(udfd)

        udfc = deepcopy(empty_udfc)

        for udfd in udfds:
            udfd_info = {
                'udfd': udfd,
                'fields': udfd.datatype_dict[0]['choices']
            }
            name_dict = udfc[to_object_name(udfd.name)]
            name_dict[to_object_name(udfd.model_type)] = udfd_info

        return udfc

    def remove_map_feature_types(self, remove=None, keep=None):
        from treemap.util import to_object_name
        if keep and remove:
            raise Exception('Invalid use of remove_map_features API: '
                            'pass arguments "keep" or "remove" but not both')
        elif keep:
            remaining_types = [name for name in self.map_feature_types
                               if name in keep]
        else:
            remaining_types = [class_name for class_name
                               in self.map_feature_types
                               if class_name not in remove]

        for class_name in self.map_feature_types:
            if class_name not in remaining_types:
                if class_name in self.search_config:
                    del self.search_config[class_name]
                if 'missing' in self.search_config:
                    self.search_config['missing'] = [
                        o for o in self.search_config['missing']
                        if not o.get('identifier', '').startswith(
                            to_object_name(class_name))]
                # TODO: delete from mobile_api_fields
                # non-plot mobile_api_fields are not currently
                # supported, but when they are added, they should
                # also be removed here.
        self.map_feature_types = remaining_types
        self.save()

def context_dict_for_resource(request, resource, **kwargs):
    context = context_dict_for_map_feature(request, resource, **kwargs)
    instance = request.instance

    # Give them 2 for adding the resource and answering its questions
    total_progress_items = 3
    completed_progress_items = 2

    context['external_link'] = None

    photos = resource.photos()
    context['photos'] = [context_dict_for_photo(request, photo)
                         for photo in photos]

    has_photos = len(photos) > 0

    if has_photos:
        completed_progress_items += 1

    context['upload_photo_endpoint'] = reverse(
        'add_photo_to_map_feature',
        kwargs={'instance_url_name': instance.url_name,
                'feature_id': resource.pk})

    context['progress_percent'] = int(100 * (
        completed_progress_items / total_progress_items) + .5)

    context['progress_messages'] = []
    if not has_photos:
        context['progress_messages'].append(_('Add a photo'))

    audits = _map_feature_audits(request.user, request.instance, resource)

    _add_audits_to_context(audits, context)

    _add_share_context(context, request, photos)

    object_name_alias = to_object_name(context['feature'].__class__.__name__)
    # some features that were originally written to support plot and tree
    # have grown to support other resource types, but they expect a context
    # entry for their type, not just for 'feature'.
    # For example:
    # * Plot detail expects 'plot' and 'tree'
    # * Foo detail would expect 'foo'
    context[object_name_alias] = context['feature']

    if isinstance(resource, PolygonalMapFeature):
        context['contained_plots'] = resource.contained_plots()
        area = resource.calculate_area()
        __, display_area = get_display_value(instance,
                                             'greenInfrastructure', 'area',
                                             area, digits=0)
        display_units = get_unit_abbreviation(
            get_units(instance, 'greenInfrastructure', 'area'))
        context['area'] = area
        context['display_area'] = display_area
        context['area_units'] = display_units

    return context

def render_map_feature_add(request, instance, type):
    if type in instance.map_feature_types[1:]:
        app = MapFeature.get_subclass(type).__module__.split('.')[0]
        try:
            template = '%s/%s_add.html' % (app, type)
        except:
            template = 'treemap/resource_add.html'
        return render(request, template, {'object_name': to_object_name(type)})
    else:
        raise_non_instance_404(type)

def render_map_feature_add(request, instance, type):
    if type in instance.map_feature_types[1:]:
        app = MapFeature.get_subclass(type).__module__.split(".")[0]
        try:
            template = "%s/%s_add.html" % (app, type)
            get_template(template)
        except:
            template = "treemap/resource_add.html"
        return render_to_response(template, {"object_name": to_object_name(type)}, RequestContext(request))
    else:
        raise_non_instance_404(type)

def _get_fields(instance, model_name):
    Model = safe_get_model_class(model_name)
    mobj = Model(instance=instance)
    udfs = {udf.canonical_name
            for udf in udf_defs(instance, model_name)
            if not udf.iscollection}
    concrete_fields = {
        f.name for f in mobj._meta.get_fields(include_parents=False)
        if _should_show_field(Model, f.name) and not f.is_relation}
    model_fields = concrete_fields | udfs
    model_fields = {'%s.%s' % (to_object_name(model_name), f)
                    for f in model_fields}
    return model_fields

def get_udfc_search_fields(instance, user):
    from treemap.models import InstanceUser
    from treemap.udf import UDFModel
    from treemap.util import to_object_name, leaf_models_of_class
    from treemap.lib.perms import udf_write_level, READ, WRITE

    try:
        iu = instance.instanceuser_set.get(user__pk=user.pk)
    except InstanceUser.DoesNotExist:
        iu = None

    data = DotDict({'models': set(), 'udfc': {}})
    for clz in leaf_models_of_class(UDFModel):
        model_name = clz.__name__
        if model_name not in ['Tree'] + instance.map_feature_types:
            continue
        for k, v in clz.collection_udf_settings.items():
            udfds = (u for u in udf_defs(instance, model_name) if u.name == k)
            for udfd in udfds:
                if udf_write_level(iu, udfd) in (READ, WRITE):
                    _base_nest_path = 'udfc.%s.' % (to_object_name(k))
                    ids_nest_path = ('%sids.%s'
                                     % (_base_nest_path,
                                        to_object_name(model_name)))
                    models_nest_path = ('%smodels.%s' %
                                        (_base_nest_path,
                                         to_object_name(model_name)))
                    data[ids_nest_path] = udfd.pk
                    data[models_nest_path] = {
                        'udfd': udfd,
                        'fields': udfd.datatype_dict[0]['choices']
                    }
                    p = 'udfc.%s.' % to_object_name(k)
                    data[p + 'action_verb'] = v['action_verb']
                    data[p + 'range_field_key'] = v['range_field_key']
                    data[p + 'action_field_key'] = v['action_field_key']
                    data['models'] |= {clz}

    return data

def render_map_feature_add(request, instance, type):
    if type in instance.map_feature_types[1:]:
        app = MapFeature.get_subclass(type).__module__.split('.')[0]
        try:
            template = '%s/%s_add.html' % (app, type)
            get_template(template)
        except:
            template = 'treemap/resource_add.html'
        return render_to_response(template,
                                  {'object_name': to_object_name(type)},
                                  RequestContext(request))
    else:
        raise Http404('Instance does not support feature type ' + type)

def context_dict_for_resource(request, resource, **kwargs):
    context = context_dict_for_map_feature(request, resource, **kwargs)
    instance = request.instance

    # Give them 2 for adding the resource and answering its questions
    total_progress_items = 3
    completed_progress_items = 2

    photos = resource.photos()
    context["photos"] = [context_dict_for_photo(request, photo) for photo in photos]

    has_photos = len(photos) > 0

    if has_photos:
        completed_progress_items += 1

    context["upload_photo_endpoint"] = reverse(
        "add_photo_to_map_feature", kwargs={"instance_url_name": instance.url_name, "feature_id": resource.pk}
    )

    context["progress_percent"] = int(100 * (completed_progress_items / total_progress_items) + 0.5)

    context["progress_messages"] = []
    if not has_photos:
        context["progress_messages"].append(_("Add a photo"))

    audits = _map_feature_audits(request.user, request.instance, resource)

    _add_audits_to_context(audits, context)

    _add_share_context(context, request, photos)

    object_name_alias = to_object_name(context["feature"].__class__.__name__)
    # some features that were originally written to support plot and tree
    # have grown to support other resource types, but they expect a context
    # entry for their type, not just for 'feature'.
    # For example:
    # * Plot detail expects 'plot' and 'tree'
    # * Foo detail would expect 'foo'
    context[object_name_alias] = context["feature"]

    if isinstance(resource, PolygonalMapFeature):
        context["contained_plots"] = resource.contained_plots()

    return context

    def _mutate_convertable_fields(self, f):
        from treemap.util import to_object_name
        # note that `to_object_name` is a helper function we use
        # for lowerCamelCase, but `._meta.object_name` is a django
        # internal that is represented as UpperCamelCase.
        model = to_object_name(self._meta.object_name)
        for field in self._meta.get_all_field_names():
            if self.instance and is_convertible(model, field):
                value = getattr(self, field)

                try:
                    value = float(value)
                except Exception:
                    # These will be caught later in the cleaning process
                    pass

                converted_value = f(self.instance, model, field, value)

                setattr(self, field, converted_value)

def save_audit(migration_rules, migration_event,
               relic_ids, model_dict, audit_obj, instance):

    fields = model_dict['fields']

    # the migrator uses downcase names
    model_name = to_object_name(fields['model'])

    model_id = relic_ids[model_name][fields['model_id']]

    if model_id == models.UNBOUND_MODEL_ID:
        print("cannot save this audit. "
              "The underlying model '%s' was discarded."
              % model_name)
        return None

    audit_obj.model_id = model_id

    if ((fields['field'] == 'id' and
         fields['current_value'] == fields['model_id'])):
        audit_obj.current_value = model_id

    # after the initial save, `created` can be updated without
    # getting clobbered by `auto_now_add`.
    # save the object, then set the created time.
    audit_obj.save()
    created = fields['created']
    assert created != '' and not created is None
    audit_obj.created = dateutil.parser.parse(created)
    audit_obj.save()

    OTM1ModelRelic.objects.create(
        instance=instance,
        migration_event=migration_event,
        otm1_model_id=model_dict['pk'],
        otm2_model_name='audit',
        otm2_model_id=audit_obj.pk)
    return audit_obj

def _add_scalar_udf_to_field_configs(udf, instance):
    save_instance = False

    for prop in ('mobile_api_fields', 'web_detail_fields'):
        attr = getattr(instance, prop)
        for group in attr:
            if (('model' in group and
                 group['model'] == to_object_name(udf.model_type))):
                field_keys = group.get('field_keys')

                if 'field_keys' in group and udf.full_name not in field_keys:
                    field_keys.append(udf.full_name)
                    save_instance = True
                    # The first time a udf is configured,
                    # getattr(instance, prop) returns a deepcopy of
                    # the default for prop.
                    # Mutating the deepcopy does not set prop on config
                    # to refer to that deepcopy, so we must do the
                    # setattr here.
                    setattr(instance, prop, attr)

    if save_instance:
        instance.save()

    def render(self, context):
        label, identifier = self.resolve_label_and_identifier(context)
        user = _resolve_variable(self.user, context)
        instance = _resolve_variable(self.instance, context)
        field_template = get_template(_resolve_variable(
                                      self.field_template, context))

        if not isinstance(identifier, basestring)\
           or not _identifier_regex.match(identifier):
            raise template.TemplateSyntaxError(
                'expected a string with the format "object_name.property" '
                'to follow "from" %s' % identifier)

        model_name_or_object_name, field_name = dotted_split(identifier, 2,
                                                             maxsplit=1)
        model = self.get_model(context, model_name_or_object_name, instance)

        object_name = to_object_name(model_name_or_object_name)
        identifier = "%s.%s" % (object_name, field_name)

        def _field_value(model, field_name):
            udf_field_name = field_name.replace('udf:', '')
            if field_name in model._meta.get_all_field_names():
                try:
                    val = getattr(model, field_name)
                except ObjectDoesNotExist:
                    val = None
            elif _is_udf(model, udf_field_name):
                val = model.udfs[udf_field_name]
            else:
                raise ValueError('Could not find field: %s' % field_name)

            return val

        if is_json_field_reference(field_name):
            field_value = get_attr_from_json_field(model, field_name)
            choices = None
            is_visible = is_editable = True
            data_type = "string"
        else:
            field_value = _field_value(model, field_name)
            data_type, label, choices = field_type_label_choices(
                model, field_name, label)

            if user is not None and hasattr(model, 'field_is_visible'):
                is_visible = model.field_is_visible(user, field_name)
                is_editable = model.field_is_editable(user, field_name)
            else:
                # This tag can be used without specifying a user. In that case
                # we assume that the content is visible and upstream code is
                # responsible for only showing the content to the appropriate
                # user
                is_visible = True
                is_editable = True

        digits = units = ''

        if hasattr(model, 'instance'):
            digits = get_digits_if_formattable(
                model.instance, object_name, field_name)

            units = get_units_if_convertible(
                model.instance, object_name, field_name)

        if field_value is None:
            display_val = None
        elif data_type == 'date' and model.instance:
            display_val = dateformat.format(field_value,
                                            model.instance.short_date_format)
        elif data_type == 'date':
            display_val = dateformat.format(field_value,
                                            settings.SHORT_DATE_FORMAT)
        elif is_convertible_or_formattable(object_name, field_name):
            display_val = format_value(
                model.instance, object_name, field_name, field_value)
            if units != '':
                display_val += (' %s' % units)
        elif data_type == 'bool':
            display_val = trans('Yes') if field_value else trans('No')
        else:
            display_val = unicode(field_value)

        context['field'] = {
            'label': label,
            'identifier': identifier,
            'value': field_value,
            'display_value': display_val,
            'units': units,
            'digits': digits,
            'data_type': data_type,
            'is_visible': is_visible,
            'is_editable': is_editable,
            'choices': choices
        }
        self.get_additional_context(context['field'])

        return field_template.render(context)

 def full_name(self):
     return to_object_name(self.model_type) + '.' + self.canonical_name

 def collection_udfs_search_names(self):
     object_name = to_object_name(self.__class__.__name__)
     return ['udf:%s:%s' % (object_name, udf.pk)
             for udf in self.collection_udfs]

    def advanced_search_fields(self):
        # TODO pull from the config once users have a way to set search fields

        from treemap.util import to_object_name

        # if we come to support more udfc searches, we can add them here.
        udfc_models = ['Tree', 'Plot']
        udfc_names = ['Stewardship', 'Alerts']

        # must inflate object so that template can look for keys
        empty_udfc = {to_object_name(n_k):
                      {to_object_name(m_k): {'fields': [], 'udfd': None}
                       for m_k in udfc_models}
                      for n_k in udfc_names}

        if not self.feature_enabled('advanced_search_filters'):
            return {'standard': [], 'missing': [], 'display': [],
                    'udfc': empty_udfc}

        from treemap.models import MapFeature  # prevent circular import
        fields = {
            'standard': [
                {'identifier': 'tree.diameter', 'search_type': 'RANGE'},
                {'identifier': 'tree.date_planted', 'search_type': 'RANGE'}
            ],
            'display': [
                {'model': 'Tree', 'label': 'Show trees'},
                {'model': 'EmptyPlot',
                 'label': 'Show empty planting sites'}
            ],
            'missing': [
                {'identifier': 'species.id',
                 'label': 'Show missing species',
                 'search_type': 'ISNULL',
                 'value': 'true'},
                {'identifier': 'tree.diameter',
                 'label': 'Show missing trunk diameter',
                 'search_type': 'ISNULL',
                 'value': 'true'},
                {'identifier': 'mapFeaturePhoto.id',
                 'label': 'Show missing photos',
                 'search_type': 'ISNULL',
                 'value': 'true'}
            ],
        }

        def make_display_filter(feature_name):
            Feature = MapFeature.get_subclass(feature_name)
            if hasattr(Feature, 'display_name_plural'):
                plural = Feature.display_name_plural
            else:
                plural = Feature.display_name + 's'
            return {
                'label': 'Show %s' % plural.lower(),
                'model': feature_name
            }

        fields['display'] += [make_display_filter(feature_name)
                              for feature_name in self.map_feature_types
                              if feature_name != 'Plot']

        # It makes styling easier if every field has an identifier
        num = 0
        for filters in fields.itervalues():
            for field in filters:
                field['id'] = "%s_%s" % (field.get('identifier', ''), num)
                num += 1

        # prevent circular import
        from treemap.udf import UserDefinedFieldDefinition

        udfds = UserDefinedFieldDefinition.objects.filter(
            instance=self,
            name__in=udfc_names,
            model_type__in=udfc_models)

        udfc = deepcopy(empty_udfc)

        for udfd in udfds:
            udfd_info = {
                'udfd': udfd,
                'fields': udfd.datatype_dict[0]['choices']
            }
            name_dict = udfc[to_object_name(udfd.name)]
            name_dict[to_object_name(udfd.model_type)] = udfd_info

        fields['udfc'] = udfc

        return fields