目标:使用 GL_TEXTURE_2D 而不是 CVOpenGLESTextureRef 将 YUV 数据(格式为 '420v' kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)推送到着色器(为什么? 因为我需要使用glTexSubImage2d来操作像素，而目标是CVOpenGLESTextureGetTarget(的情况下不能使用，所以没有效果，必须使用GL_TEXTURE_2D)
问题: 我正在使用自定义视频合成器来操作 AVPlayer 视频。当我在 Apple 的 AVCustomEdit 示例代码中使用 CVOpenGLESTextureRef 时，它使用 2 个单独的着色器，一个用于 Y(亮度)，一个用于 UV(色度)，视频看起来像这样正常:
但如果尝试使用 GL_TEXTURE_2D 来代替，视频只会显示绿色和粉红色，如下所示:
如果我将 GL_TEXTURE_2D 与结合了 Y 和 UV 纹理的片段着色器一起使用，它看起来会更糟:
我的代码:
首先创建轨道缓冲区和目标缓冲区:
CVPixelBufferRef foregroundSourceBuffer = [request sourceFrameByTrackID:currentInstruction.foregroundTrackID];
CVPixelBufferRef dstBuffer = [_renderContext newPixelBuffer];
然后它们被传递给包含以下相关代码的渲染函数:
CVOpenGLESTextureRef foregroundLumaTexture = [self lumaTextureForPixelBuffer:foregroundPixelBuffer];
CVOpenGLESTextureRef foregroundChromaTexture = [self chromaTextureForPixelBuffer:foregroundPixelBuffer];
CVOpenGLESTextureRef destLumaTexture = [self lumaTextureForPixelBuffer:destinationPixelBuffer];
CVOpenGLESTextureRef destChromaTexture = [self chromaTextureForPixelBuffer:destinationPixelBuffer];
亮度纹理函数返回:
CVOpenGLESTextureRef luma = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RED_EXT,
(int)CVPixelBufferGetWidth(pixelBuffer),
(int)CVPixelBufferGetHeight(pixelBuffer),
GL_RED_EXT,
GL_UNSIGNED_BYTE,
0,
&lumaTexture);
色度纹理函数返回:
CVOpenGLESTextureRef chroma = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RG_EXT,
(int)CVPixelBufferGetWidthOfPlane(pixelBuffer, 1),
(int)CVPixelBufferGetHeightOfPlane(pixelBuffer, 1),
GL_RG_EXT,
GL_UNSIGNED_BYTE,
1,
&chromaTexture);
现在渲染函数的相关主体:
glBindFramebuffer(GL_FRAMEBUFFER, self.offscreenBufferHandle);
glViewport(0, 0, (int)CVPixelBufferGetWidthOfPlane(destinationPixelBuffer, 0), (int)CVPixelBufferGetHeightOfPlane(destinationPixelBuffer, 0));
#ifdef USE_GL_TEXTURE_2D
int bufferWidth = CVPixelBufferGetWidth(foregroundPixelBuffer);
int bufferHeight = CVPixelBufferGetHeight(foregroundPixelBuffer);
GLuint frameTextureY;
GLuint frameTextureUV;
glGenTextures(1, &frameTextureY);
glGenTextures(1, &frameTextureUV);
if(CVPixelBufferLockBaseAddress(foregroundPixelBuffer, 0) == kCVReturnSuccess){
glBindTexture(GL_TEXTURE_2D, frameTextureY);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, bufferWidth, bufferHeight, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddressOfPlane(foregroundPixelBuffer, 0));
glBindTexture(GL_TEXTURE_2D, frameTextureUV);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, bufferWidth/2, bufferHeight/2, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddressOfPlane(foregroundPixelBuffer, 1));
CVPixelBufferUnlockBaseAddress(foregroundPixelBuffer, 0);
}
#endif
glActiveTexture(GL_TEXTURE0);
#ifdef USE_GL_TEXTURE_2D
glUseProgram(self.programYUV_2);
glBindTexture(GL_TEXTURE_2D, frameTextureY);
glUniformMatrix4fv(uniforms[UNIFORM_RENDER_TRANSFORM_YUV_2], 1, GL_FALSE, preferredRenderTransform);
#else
glUseProgram(self.programY);
glBindTexture(CVOpenGLESTextureGetTarget(foregroundLumaTexture), CVOpenGLESTextureGetName(foregroundLumaTexture));
glUniformMatrix4fv(uniforms[UNIFORM_RENDER_TRANSFORM_Y], 1, GL_FALSE, preferredRenderTransform);
#endif
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// Attach the destination texture as a color attachment to the off screen frame buffer
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, CVOpenGLESTextureGetTarget(destLumaTexture), CVOpenGLESTextureGetName(destLumaTexture), 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
NSLog(@"Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
goto bail;
}
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
#ifdef USE_GL_TEXTURE_2D
glUniform1i(uniforms[UNIFORM_TEXTURE_YUV_2_Y], 0);
glVertexAttribPointer(ATTRIB_VERTEX_Y_UV_INONESHADER, 2, GL_FLOAT, 0, 0, quadVertexData1);
glEnableVertexAttribArray(ATTRIB_VERTEX_Y_UV_INONESHADER);
glVertexAttribPointer(ATTRIB_TEXCOORD_Y_UV_INONESHADER, 2, GL_FLOAT, 0, 0, quadTextureData1);
glEnableVertexAttribArray(ATTRIB_TEXCOORD_Y_UV_INONESHADER);
#else
glUniform1i(uniforms[UNIFORM_TEXTURE_Y], 0);
glVertexAttribPointer(ATTRIB_VERTEX_Y, 2, GL_FLOAT, 0, 0, quadVertexData1);
glEnableVertexAttribArray(ATTRIB_VERTEX_Y);
glVertexAttribPointer(ATTRIB_TEXCOORD_Y, 2, GL_FLOAT, 0, 0, quadTextureData1);
glEnableVertexAttribArray(ATTRIB_TEXCOORD_Y);
#endif
glDrawArrays(GL_TRIANGLE_STRIP, 0, 5);
glActiveTexture(GL_TEXTURE1);
#ifdef USE_GL_TEXTURE_2D
//no need to use different program
glBindTexture(GL_TEXTURE_2D, frameTextureUV);
glUniformMatrix4fv(uniforms[UNIFORM_RENDER_TRANSFORM_YUV_2], 1, GL_FALSE, preferredRenderTransform);
#else
glUseProgram(self.programUV);
glBindTexture(CVOpenGLESTextureGetTarget(foregroundChromaTexture), CVOpenGLESTextureGetName(foregroundChromaTexture));
glUniformMatrix4fv(uniforms[UNIFORM_RENDER_TRANSFORM_UV], 1, GL_FALSE, preferredRenderTransform);
#endif
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glViewport(0, 0, (int)CVPixelBufferGetWidthOfPlane(destinationPixelBuffer, 1), (int)CVPixelBufferGetHeightOfPlane(destinationPixelBuffer, 1));
// Attach the destination texture as a color attachment to the off screen frame buffer
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, CVOpenGLESTextureGetTarget(destChromaTexture), CVOpenGLESTextureGetName(destChromaTexture), 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
NSLog(@"Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
goto bail;
}
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
#ifdef USE_GL_TEXTURE_2D
glUniform1i(uniforms[UNIFORM_TEXTURE_YUV_2_UV], 1);
glVertexAttribPointer(ATTRIB_VERTEX_Y_UV_INONESHADER, 2, GL_FLOAT, 0, 0, quadVertexData1);
glEnableVertexAttribArray(ATTRIB_VERTEX_Y_UV_INONESHADER);
glVertexAttribPointer(ATTRIB_TEXCOORD_Y_UV_INONESHADER, 2, GL_FLOAT, 0, 0, quadTextureData1);
glEnableVertexAttribArray(ATTRIB_TEXCOORD_Y_UV_INONESHADER);#else
glUniform1i(uniforms[UNIFORM_TEXTURE_UV], 1);
glVertexAttribPointer(ATTRIB_VERTEX_UV, 2, GL_FLOAT, 0, 0, quadVertexData1);
glEnableVertexAttribArray(ATTRIB_VERTEX_UV);
glVertexAttribPointer(ATTRIB_TEXCOORD_UV, 2, GL_FLOAT, 0, 0, quadTextureData1);
glEnableVertexAttribArray(ATTRIB_TEXCOORD_UV);
#endif
glDrawArrays(GL_TRIANGLE_STRIP, 0, 5);
glFlush();
bail:
#ifdef USE_GL_TEXTURE_2D
glDeleteTextures(1, &frameTextureY);
glDeleteTextures(1, &frameTextureUV);
#endif
CFRelease(foregroundLumaTexture);
CFRelease(foregroundChromaTexture);
CFRelease(destLumaTexture);
CFRelease(destChromaTexture);
// Periodic texture cache flush every frame
CVOpenGLESTextureCacheFlush(self.videoTextureCache, 0);
这是我的片段着色器，我根据不同的测试用例使用(无论是单独绘制 Y 和 UV 还是一起绘制):
static const char kFragmentShaderY[] = {
"varying highp vec2 texCoordVarying; \n \
uniform sampler2D s_texture_y; \n \
void main() \n \
{ \n \
gl_FragColor.r = texture2D(s_texture_y, texCoordVarying).r; \n \
}"
};
static const char kFragmentShaderUV[] = {
"varying highp vec2 texCoordVarying; \n \
uniform sampler2D s_texture_uv; \n \
void main() \n \
{ \n \
gl_FragColor.rg = texture2D(s_texture_uv, texCoordVarying).rg; \n \
}"
};
static const char kFragmentShaderYUV_2Textures[] = {
"varying highp vec2 texCoordVarying; \n \
uniform sampler2D s_texture_y; \n \
uniform sampler2D s_texture_uv; \n \
\n \
void main() \n \
{ \n \
mediump vec3 yuv;// = vec3(1.1643 * (texture2D(s_texture_y, texCoordVarying).r - 0.0625), \n \
lowp vec3 rgb; \n \
yuv.x = texture2D(s_texture_y, texCoordVarying).r; \n \
yuv.yz = texture2D(s_texture_uv, texCoordVarying).rg - vec2(0.5, 0.5); \n \
\n \
rgb = mat3( 1, 1, 1, \n \
0, -.21482, 2.12798, \n \
1.28033, -.38059, 0) * yuv; \n \
gl_FragColor = vec4(rgb, 1.0); \n \
}"
};
使用 GL_TEXTURE_2D，如果我使用包含 Y 和 UV 纹理的片段着色器，视频看起来像上面的 #3。如果我使用两个单独的片段着色器(一个用于 Y，一个用于 UV)，图片是上面的 #2(几乎正确，但色度颜色只是绿色和粉红色)*(请注意，我确实注释掉了上面的一些代码能够使用 2 个单独的片段着色器，当然我 glBind 到 GL_TEXTURE_2D 而不是 CV，依此类推，依此类推)。
再次，我的问题是我需要使用 GL_TEXTURE_2D 而不是 CVOpenGLESTextureGetTarget，但如果我这样做，它不会显示正确的色度颜色。我想知道我做错了什么。可能与 YUV 格式 kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange 而不是 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange 有关吗？我还尝试了制作 3 个 GL_LUMINANCE 纹理的方法，以及许多其他没有运气的排列。
事实证明，问题在于使用 GL_LUMINANCE 和 GL_LUMINANCE_ALPHA，这显然是不推荐使用的格式。当我将它们切换到 GL_RED_EXT 和 GL_RG_EXT 时，它起作用了，色度颜色终于正确了。我希望这个问题和答案可以节省其他人的时间。
关于使用 GL_TEXTURE_2D 的 iOS YUV 420v 在 OpenGL 着色器中显示错误的颜色，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/37091132/
| 欢迎光临 OStack程序员社区-中国程序员成长平台 (https://ostack.cn/) | Powered by Discuz! X3.4 |