def value_compare(l, r, ordering=1):
    if l == None:
        if r == None:
            return 0
        else:
            return - ordering
    elif r == None:
        return ordering

    if isinstance(l, list) or isinstance(r, list):
        for a, b in zip(listwrap(l), listwrap(r)):
            c = value_compare(a, b) * ordering
            if c != 0:
                return c
        return 0
    elif isinstance(l, Mapping):
        if isinstance(r, Mapping):
            for k in set(l.keys()) | set(r.keys()):
                c = value_compare(l[k], r[k]) * ordering
                if c != 0:
                    return c
            return 0
        else:
            return 1
    elif isinstance(r, Mapping):
        return -1
    else:
        return cmp(l, r) * ordering

    def error(
        cls,
        template,  # human readable template
        default_params={},  # parameters for template
        cause=None,  # pausible cause
        stack_depth=0,
        **more_params
    ):
        """
        raise an exception with a trace for the cause too
        """
        if default_params and isinstance(listwrap(default_params)[0], BaseException):
            cause = default_params
            default_params = {}

        params = dict(unwrap(default_params), **more_params)

        add_to_trace = False
        cause = unwraplist([Except.wrap(c, stack_depth=1) for c in listwrap(cause)])
        trace = extract_stack(stack_depth + 1)

        if add_to_trace:
            cause[0].trace.extend(trace[1:])

        e = Except(ERROR, template, params, cause, trace)
        raise e

    def error(
        cls,
        template,  # human readable template
        default_params={},  # parameters for template
        cause=None,  # pausible cause
        stack_depth=0,
        **more_params
    ):
        """
        raise an exception with a trace for the cause too

        :param template: *string* human readable string with placeholders for parameters
        :param default_params: *dict* parameters to fill in template
        :param cause: *Exception* for chaining
        :param stack_depth:  *int* how many calls you want popped off the stack to report the *true* caller
        :param log_context: *dict* extra key:value pairs for your convenience
        :param more_params: *any more parameters (which will overwrite default_params)
        :return:
        """
        if default_params and isinstance(listwrap(default_params)[0], BaseException):
            cause = default_params
            default_params = {}

        params = dict(unwrap(default_params), **more_params)

        add_to_trace = False
        cause = wrap(unwraplist([Except.wrap(c, stack_depth=1) for c in listwrap(cause)]))
        trace = exceptions.extract_stack(stack_depth + 1)

        if add_to_trace:
            cause[0].trace.extend(trace[1:])

        e = Except(exceptions.ERROR, template, params, cause, trace)
        raise e

    def map(self, map_):
        def map_select(s, map_):
            return set_default({"value": s.value.map(map_)}, s)

        def map_edge(e, map_):
            partitions = unwraplist([set_default({"where": p.where.map(map_)}, p) for p in e.domain.partitions])

            domain = copy(e.domain)
            domain.where = e.domain.where.map(map_)
            domain.partitions = partitions

            edge = copy(e)
            edge.value = e.value.map(map_)
            edge.domain = domain
            if e.range:
                edge.range.min = e.range.min.map(map_)
                edge.range.max = e.range.max.map(map_)
            return edge

        return QueryOp(
            "from",
            frum=self.frum.map(map_),
            select=wrap([map_select(s, map_) for s in listwrap(self.select)]),
            edges=wrap([map_edge(e, map_) for e in self.edges]),
            groupby=wrap([g.map(map_) for g in self.groupby]),
            window=wrap([w.map(map_) for w in self.window]),
            where=self.where.map(map_),
            sort=wrap([map_select(s, map_) for s in listwrap(self.sort)]),
            limit=self.limit,
            format=self.format,
        )

def argparse(defs):
    parser = _argparse.ArgumentParser()
    for d in listwrap(defs):
        args = d.copy()
        name = args.name
        args.name = None
        parser.add_argument(*unwrap(listwrap(name)), **args)
    namespace = parser.parse_args()
    output = {k: getattr(namespace, k) for k in vars(namespace)}
    return wrap(output)

def get_all_vars(query):
    output = []
    for s in listwrap(query.select):
        output.extend(select_get_all_vars(s))
    for s in listwrap(query.edges):
        output.extend(edges_get_all_vars(s))
    for s in listwrap(query.groupby):
        output.extend(edges_get_all_vars(s))
    output.extend(expressions.get_all_vars(query.where))
    return output

    def _set_op(self, query):
        if listwrap(query.select)[0].value == ".":
            selects = ["*"]
        else:
            selects = [s.value.to_sql() + " AS " + quote_table(s.name) for s in listwrap(query.select)]

        for w in query.window:
            selects.append(self._window_op(self, query, w))

        agg = " FROM " + quote_table(self.name) + " a"
        where = "\nWHERE " + query.where.to_sql()

        return "SELECT " + (",\n".join(selects)) + agg + where

        def errors(e, _buffer):  # HANDLE ERRORS FROM extend()

            if e.cause.cause:
                not_possible = [f for f in listwrap(e.cause.cause) if "JsonParseException" in f or "400 MapperParsingException" in f]
                still_have_hope = [f for f in listwrap(e.cause.cause) if "JsonParseException" not in f and "400 MapperParsingException" not in f]
            else:
                not_possible = [e]
                still_have_hope = []

            if still_have_hope:
                Log.warning("Problem with sending to ES", cause=still_have_hope)
            elif not_possible:
                # THERE IS NOTHING WE CAN DO
                Log.warning("Not inserted, will not try again", cause=not_possible[0:10:])
                del _buffer[:]

def value_compare(l, r, ordering=1):
    """
    SORT VALUES, NULL IS THE LEAST VALUE
    :param l: LHS
    :param r: RHS
    :param ordering: (-1, 0, 0) TO AFFECT SORT ORDER
    :return: The return value is negative if x < y, zero if x == y and strictly positive if x > y.
    """

    if l == None:
        if r == None:
            return 0
        else:
            return ordering
    elif r == None:
        return - ordering

    if isinstance(l, list) or isinstance(r, list):
        for a, b in zip(listwrap(l), listwrap(r)):
            c = value_compare(a, b) * ordering
            if c != 0:
                return c

        if len(l) < len(r):
            return - ordering
        elif len(l) > len(r):
            return ordering
        else:
            return 0
    elif isinstance(l, builtin_tuple) and isinstance(r, builtin_tuple):
        for a, b in zip(l, r):
            c = value_compare(a, b) * ordering
            if c != 0:
                return c
        return 0
    elif isinstance(l, Mapping):
        if isinstance(r, Mapping):
            for k in sorted(set(l.keys()) | set(r.keys())):
                c = value_compare(l.get(k), r.get(k)) * ordering
                if c != 0:
                    return c
            return 0
        else:
            return 1
    elif isinstance(r, Mapping):
        return -1
    else:
        return cmp(l, r) * ordering

    def select(self, select):
        selects = listwrap(select)

        if not all(isinstance(s.value, Variable) for s in selects):
            Log.error("selecting on structure, or expressions, not supported yet")
        if len(selects) == 1 and isinstance(selects[0].value, Variable) and selects[0].value.var == ".":
            new_schema = self.schema
            if selects[0].name == ".":
                return self
        else:
            new_schema = None

        if isinstance(select, list):
            push_and_pull = [(s.name, jx_expression_to_function(s.value)) for s in selects]
            def selector(d):
                output = Dict()
                for n, p in push_and_pull:
                    output[n] = p(wrap(d))
                return unwrap(output)

            new_data = map(selector, self.data)
        else:
            select_value = jx_expression_to_function(select.value)
            new_data = map(select_value, self.data)

        return ListContainer("from "+self.name, data=new_data, schema=new_schema)

def is_terms(query):
    select = listwrap(query.select)

    isSimple = not query.select or AND(aggregates[s.aggregate] in ("none", "count") for s in select)
    if isSimple:
        return True
    return False

    def warning(
        cls,
        template,
        default_params={},
        cause=None,
        stack_depth=0,       # stack trace offset (==1 if you do not want to report self)
        **more_params
    ):
        if isinstance(default_params, BaseException):
            cause = default_params
            default_params = {}

        params = dict(unwrap(default_params), **more_params)
        cause = unwraplist([Except.wrap(c) for c in listwrap(cause)])
        trace = extract_stack(stack_depth + 1)

        e = Except(WARNING, template, params, cause, trace)
        Log.note(
            unicode(e),
            {
                "warning": {# REDUNDANT INFO
                    "template": template,
                    "params": params,
                    "cause": cause,
                    "trace": trace
                }
            },
            stack_depth=stack_depth + 1
        )

def split_expression_by_depth(where, schema, map_, output=None, var_to_depth=None):
    """
    It is unfortunate that ES can not handle expressions that
    span nested indexes.  This will split your where clause
    returning {"and": [filter_depth0, filter_depth1, ...]}
    """
    vars_ = where.vars()

    if var_to_depth is None:
        if not vars_:
            return Null
        # MAP VARIABLE NAMES TO HOW DEEP THEY ARE
        var_to_depth = {v: len(listwrap(schema[v].nested_path)) for v in vars_}
        all_depths = set(var_to_depth.values())
        output = wrap([[] for _ in range(MAX(all_depths) + 1)])
    else:
        all_depths = set(var_to_depth[v] for v in vars_)

    if len(all_depths) == 1:
        output[list(all_depths)[0]] += [where.map(map_)]
    elif isinstance(where, AndOp):
        for a in where.terms:
            split_expression_by_depth(a, schema, map_, output, var_to_depth)
    else:
        Log.error("Can not handle complex where clause")

    return output

def query_get_all_vars(query, exclude_where=False):
    """
    :param query:
    :param exclude_where: Sometimes we do not what to look at the where clause
    :return: all variables in use by query
    """
    output = set()
    for s in listwrap(query.select):
        output |= select_get_all_vars(s)
    for s in listwrap(query.edges):
        output |= edges_get_all_vars(s)
    for s in listwrap(query.groupby):
        output |= edges_get_all_vars(s)
    if not exclude_where:
        output |= qb_expression(query.where).vars()
    return output

def _get_schema_from_list(frum, columns, prefix, nested_path):
    """
    SCAN THE LIST FOR COLUMN TYPES
    """
    names = {}
    for d in frum:
        for name, value in d.items():
            agg_type = names.get(name, "undefined")
            this_type = _type_to_name[value.__class__]
            new_type = _merge_type[agg_type][this_type]
            names[name] = new_type

            if this_type == "object":
                _get_schema_from_list([value], columns, prefix + [name], nested_path)
            elif this_type == "nested":
                np = listwrap(nested_path)
                newpath = unwraplist([".".join((np[0], name))]+np)
                _get_schema_from_list(value, columns, prefix + [name], newpath)

    for n, t in names.items():
        full_name = ".".join(prefix + [n])
        column = Column(
            table=".",
            name=full_name,
            abs_name=full_name,
            type=t,
            nested_path=nested_path
        )
        columns[column.name] = column

 def __getitem__(self, item):
     for s in listwrap(self.cube.select):
         if s.name == item:
             return self.cube.data[item]
     for i, e in enumerate(self.cube.edges):
         if e.name == item:
             return e.domain.partition[self.coord[i]]

def is_aggsop(es, query):
    es.cluster.get_metadata()
    if any(map(es.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])) and (
        query.edges or query.groupby or any(a != None and a != "none" for a in listwrap(query.select).aggregate)
    ):
        return True
    return False

def get_decoders_by_depth(query):
    """
    RETURN A LIST OF DECODER ARRAYS, ONE ARRAY FOR EACH NESTED DEPTH
    """
    schema = query.frum
    output = DictList()
    for e in coalesce(query.edges, query.groupby, []):
        if e.value:
            e = e.copy()
            e.value = qb_expression(e.value)
            vars_ = e.value.vars()

            for v in vars_:
                if not schema[v]:
                    Log.error("{{var}} does not exist in schema", var=v)

            e.value = e.value.map({schema[v].name: schema[v].abs_name for v in vars_})
        else:
            vars_ = e.domain.dimension.fields
            e.domain.dimension = e.domain.dimension.copy()
            e.domain.dimension.fields = [schema[v].abs_name for v in vars_]

        depths = set(len(listwrap(schema[v].nested_path)) for v in vars_)
        if len(depths) > 1:
            Log.error("expression {{expr}} spans tables, can not handle", expr=e.value)
        depth = list(depths)[0]
        while len(output) <= depth:
            output.append([])
        output[depth].append(AggsDecoder(e, query))
    return output

def es_countop(es, mvel, query):
    """
    RETURN SINGLE COUNT
    """
    select = listwrap(query.select)
    FromES = build_es_query(query)
    for s in select:

        if is_keyword(s.value):
            FromES.facets[s.name] = {
                "terms": {
                    "field": s.value,
                    "size": query.limit,
                },
                "facet_filter":{"exists":{"field":s.value}}
            }
        else:
            # COMPLICATED value IS PROBABLY A SCRIPT, USE IT
            FromES.facets[s.name] = {
                "terms": {
                    "script_field": es09.expressions.compile_expression(s.value, query),
                    "size": 200000
                }
            }

    data = es09.util.post(es, FromES, query.limit)

    matricies = {}
    for s in select:
        matricies[s.name] = Matrix(value=data.hits.facets[s.name].total)

    cube = Cube(query.select, query.edges, matricies)
    cube.frum = query
    return cube

def _normalize_groupby(groupby, schema=None):
    if groupby == None:
        return None
    output = wrap([_normalize_group(e, schema=schema) for e in listwrap(groupby)])
    if any(o==None for o in output):
        Log.error("not expected")
    return output