opengl - Unity Shader highlighting overlaps

Question

I am trying to write a shader for unity that will highlight the overlapping fragments of meshes. It should work for one object overlapping itself as well as multiple objects.

The result should look like an attached image.

First I tried to accomplish this with collision detection but I think that the best way is writing a shader.

I'm not very familiar with shaders so if anyone could help me I would be grateful.

I think that it can be done by using stencil shaders like here http://docs.unity3d.com/Manual/SL-Stencil.html but this shaders only render intersection of two objects without rendering whole object.

I also found shader based on Depth (https://chrismflynn.wordpress.com/2012/09/06/fun-with-shaders-and-the-depth-buffer/) but this also work on two objects and doesn't work on one mesh that overlap itself

Regarding @Zze comment with idea about two Pass I have now two shaders. And it works on two objects when one have one shader and other have second one.

Maybe any one can help me how to combine it into one shader that will work also in object that will overlap itself?

ShaderOne

Shader "Custom/ShaderOne"
{
    SubShader {
        Tags { "RenderType"="Opaque" "Queue"="Geometry"}
        Pass {
            Stencil {
                Ref 2
                Comp always
                Pass keep 
                Fail decrWrap 
                ZFail keep
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            struct appdata {
                float4 vertex : POSITION;
            };
            struct v2f {
                float4 pos : SV_POSITION;
            };
            v2f vert(appdata v) {
                v2f o;
                o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
                return o;
            }
            half4 frag(v2f i) : SV_Target {
                return half4(0,1,0,1);
            }
            ENDCG
        }
        Pass {
            Stencil {
                Ref 2
                Comp equal
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            struct appdata {
                float4 vertex : POSITION;
            };
            struct v2f {
                float4 pos : SV_POSITION;
            };
            v2f vert(appdata v) {
                v2f o;
                o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
                return o;
            }
            half4 frag(v2f i) : SV_Target {
                return half4(0,0,1,1);
            }
            ENDCG
        }

    } 
}

ShaderTwo

Shader "Custom/ShaderTwo"
{
    SubShader {
        Tags { "RenderType"="Opaque" "Queue"="Geometry"}
        Pass {
            Stencil {
                Ref 2
                Comp always
                Pass replace
                ZFail keep
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag
            struct appdata {
                float4 vertex : POSITION;
            };
            struct v2f {
                float4 pos : SV_POSITION;
            };
            v2f vert(appdata v) {
                v2f o;
                o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
                return o;
            }
            half4 frag(v2f i) : SV_Target {
                return half4(1,0,0,1);
            }
            ENDCG
        }
    } 
}

The result looks like an attached image

Answer 1

This problem can be solved with help of Stencil buffer and one, two-pass shader. The idea is the following:

• First pass compares the value in stencil buffer with 0. In both cases (pass/fail) increase the value in buffer.
• Second pass compares the value in stencil buffer with 1. If reference value 1 is less, than we pass and highlight the overlapping pixel.

You might want to add more passes that are the same as the second, but with different reference value to highlight regions that overlaps two times, three times, etc.

In Unity's shaderlab notation, it should be something like that:

    Pass
    {
        Stencil {
            Ref 0
            Comp Equal
            Pass IncrSat 
            Fail IncrSat 
        }

        // Shader for not overlapping regions goes here.
    }

    Pass
    {
        Stencil {
            Ref 1
            Comp Less
        }

        // Shader for one-time overlapping regions goes here.
    }

    Pass
    {
        Stencil {
            Ref 2
            Comp Less
        }

        // Shader for two-time overlapping regions goes here.
    }

Example:

Shader:

Shader "Unlit/Stencil"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
    }
    SubShader
    {
        Tags { "RenderType"="Opaque" }
        LOD 100

        Pass
        {
            Stencil {
                Ref 0
                Comp Equal
                Pass IncrSat 
                Fail IncrSat 
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
            };

            struct v2f
            {
                float4 vertex : SV_POSITION;
            };

            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
                return o;
            }

            fixed4 frag (v2f i) : SV_Target
            {
                fixed4 col = fixed4(0.0, 0.0, 1.0, 1.0);
                return col;
            }
            ENDCG
        }

        Pass
        {
            Stencil {
                Ref 1
                Comp Less
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
            };

            struct v2f
            {
                float4 vertex : SV_POSITION;
            };

            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
                return o;
            }

            fixed4 frag (v2f i) : SV_Target
            {
                fixed4 col = fixed4(1.0, 1.0, 0.0, 1.0);
                return col;
            }
            ENDCG
        }

        Pass
        {
            Stencil {
                Ref 2
                Comp Less
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
            };

            struct v2f
            {
                float4 vertex : SV_POSITION;
            };

            v2f vert (appdata v)
            {
                v2f o;
                o.vertex = mul(UNITY_MATRIX_MVP, v.vertex);
                return o;
            }

            fixed4 frag (v2f i) : SV_Target
            {
                fixed4 col = fixed4(1.0, 0.0, 0.0, 1.0);
                return col;
            }
            ENDCG
        }
    }
}