Ported: fix post procressing with transparent bg and keep alpha channel of the tonemap shader output. - Kinwailo

b1a50ad805

drivers/gles2/rasterizer_scene_gles2.cpp

@@ -2708,9 +2708,8 @@ void RasterizerSceneGLES2::_post_process(Environment *env, const CameraMatrix &p
 	glDepthFunc(GL_LEQUAL);
 	glColorMask(1, 1, 1, 1);
 
-	//no post process on small, transparent or render targets without an env
-	bool use_post_process = env && !storage->frame.current_rt->flags[RasterizerStorage::RENDER_TARGET_TRANSPARENT];
-	use_post_process = use_post_process && storage->frame.current_rt->width >= 4 && storage->frame.current_rt->height >= 4;
+	//no post process on small or render targets without an env
+	bool use_post_process = env && storage->frame.current_rt->width >= 4 && storage->frame.current_rt->height >= 4;
 	use_post_process = use_post_process && storage->frame.current_rt->mip_maps_allocated;
 
 	if (env) {

drivers/gles2/shaders/effect_blur.glsl

@@ -250,7 +250,7 @@ void main() {
 #endif
 
 	float amount = smoothstep(dof_begin, dof_end, depth);
-	float k_accum = 0.0;
+  vec4 k_accum = vec4(0.0);
 
 	for (int i = 0; i < dof_kernel_size; i++) {
 		int int_ofs = i - dof_kernel_from;
@@ -267,14 +267,16 @@ void main() {
 #endif
 		float tap_amount = int_ofs == 0 ? 1.0 : smoothstep(dof_begin, dof_end, tap_depth);
 		tap_amount *= tap_amount * tap_amount; //prevent undesired glow effect
+    tap_amount *= tap_k;
 
-		vec4 tap_color = texture2DLod(source_color, tap_uv, 0.0) * tap_k;
+    vec4 tap_color = texture2DLod(source_color, tap_uv, 0.0);
 
-		k_accum += tap_k * tap_amount;
-		color_accum += tap_color * tap_amount;
+    vec4 w = vec4(tap_amount) * vec4(vec3(tap_color.a), 1.0);
+		k_accum += w;
+		color_accum += tap_color * w;
 	}
 
-	if (k_accum > 0.0) {
+	if (k_accum.r > 0.0) {
 		color_accum /= k_accum;
 	}
 
@@ -287,6 +289,7 @@ void main() {
 	vec4 color_accum = vec4(0.0);
 
 	float max_accum = 0.0;
+  float k_accum = 0.0;
 
 	for (int i = 0; i < dof_kernel_size; i++) {
 		int int_ofs = i - dof_kernel_from;
@@ -296,6 +299,7 @@ void main() {
 		float tap_k = dof_kernel[i];
 
 		vec4 tap_color = texture2DLod(source_color, tap_uv, 0.0);
+    float w = tap_color.a;
 
 		float tap_depth = texture2D(dof_source_depth, tap_uv, 0.0).r;
 		tap_depth = tap_depth * 2.0 - 1.0;
@@ -314,10 +318,16 @@ void main() {
 #endif
 
 		max_accum = max(max_accum, tap_amount * ofs_influence);
+    k_accum += w;
+		tap_color.rgb *= w;
 
 		color_accum += tap_color * tap_k;
 	}
 
+  if (k_accum > 0.0) {
+		color_accum.rgb /= k_accum / dof_kernel_size;
+	}
+
 	color_accum.a = max(color_accum.a, sqrt(max_accum));
 
 	gl_FragColor = color_accum;

drivers/gles2/shaders/tonemap.glsl

@@ -180,13 +180,13 @@ vec4 texture2D_bicubic(sampler2D tex, vec2 uv, int p_lod) {
 #define GLOW_TEXTURE_SAMPLE(m_tex, m_uv, m_lod) texture2DLod(m_tex, m_uv, float(m_lod))
 #endif //GL_EXT_gpu_shader4
 
-vec3 apply_glow(vec3 color, vec3 glow) { // apply glow using the selected blending mode
+vec4 apply_glow(vec4 color, vec3 glow) { // apply glow using the selected blending mode
 #ifdef USE_GLOW_REPLACE
-	color = glow;
+  color.rgb = glow;
 #endif
 
 #ifdef USE_GLOW_SCREEN
-	color = max((color + glow) - (color * glow), vec3(0.0));
+  color.rgb = max((color.rgb + glow) - (color.rgb * glow), vec3(0.0));
 #endif
 
 #ifdef USE_GLOW_SOFTLIGHT
@@ -198,7 +198,18 @@ vec3 apply_glow(vec3 color, vec3 glow) { // apply glow using the selected blendi
 #endif
 
 #if !defined(USE_GLOW_SCREEN) && !defined(USE_GLOW_SOFTLIGHT) && !defined(USE_GLOW_REPLACE) // no other selected -> additive
-	color += glow;
+	color.rgb += glow;
+#endif
+
+#ifndef USE_GLOW_SOFTLIGHT // softlight has no effect on black color
+	// compute the alpha from glow
+	float a = max(max(glow.r, glow.g), glow.b);
+	color.a = a + color.a * (1 - a);
+	if (color.a == 0.0) {
+		color.rgb = vec3(0.0);
+	} else if (color.a < 1.0) {
+		color.rgb /= color.a;
+	}
 #endif
 
 	return color;
@@ -220,22 +231,22 @@ vec3 apply_color_correction(vec3 color, sampler2D correction_tex) {
 	return color;
 }
 
-vec3 apply_fxaa(vec3 color, vec2 uv_interp, vec2 pixel_size) {
+vec4 apply_fxaa(vec4 color, vec2 uv_interp, vec2 pixel_size) {
 	const float FXAA_REDUCE_MIN = (1.0 / 128.0);
 	const float FXAA_REDUCE_MUL = (1.0 / 8.0);
 	const float FXAA_SPAN_MAX = 8.0;
 
-	vec3 rgbNW = texture2DLod(source, uv_interp + vec2(-1.0, -1.0) * pixel_size, 0.0).xyz;
-	vec3 rgbNE = texture2DLod(source, uv_interp + vec2(1.0, -1.0) * pixel_size, 0.0).xyz;
-	vec3 rgbSW = texture2DLod(source, uv_interp + vec2(-1.0, 1.0) * pixel_size, 0.0).xyz;
-	vec3 rgbSE = texture2DLod(source, uv_interp + vec2(1.0, 1.0) * pixel_size, 0.0).xyz;
-	vec3 rgbM = color;
+	vec4 rgbNW = texture2DLod(source, uv_interp + vec2(-1.0, -1.0) * pixel_size, 0.0);
+	vec4 rgbNE = texture2DLod(source, uv_interp + vec2(1.0, -1.0) * pixel_size, 0.0);
+	vec4 rgbSW = texture2DLod(source, uv_interp + vec2(-1.0, 1.0) * pixel_size, 0.0);
+	vec4 rgbSE = texture2DLod(source, uv_interp + vec2(1.0, 1.0) * pixel_size, 0.0);
+	vec3 rgbM = color.rgb;
 	vec3 luma = vec3(0.299, 0.587, 0.114);
-	float lumaNW = dot(rgbNW, luma);
-	float lumaNE = dot(rgbNE, luma);
-	float lumaSW = dot(rgbSW, luma);
-	float lumaSE = dot(rgbSE, luma);
-	float lumaM = dot(rgbM, luma);
+	float lumaNW = dot(rgbNW.rgb, luma) - ((1 - rgbNW.a) / 8.0);
+	float lumaNE = dot(rgbNE.rgb, luma) - ((1 - rgbNE.a) / 8.0);
+	float lumaSW = dot(rgbSW.rgb, luma) - ((1 - rgbSW.a) / 8.0);
+	float lumaSE = dot(rgbSE.rgb, luma) - ((1 - rgbSE.a) / 8.0);
+	float lumaM = dot(rgbM, luma) - (color.a / 8.0);
 	float lumaMin = min(lumaM, min(min(lumaNW, lumaNE), min(lumaSW, lumaSE)));
 	float lumaMax = max(lumaM, max(max(lumaNW, lumaNE), max(lumaSW, lumaSE)));
 
@@ -253,19 +264,22 @@ vec3 apply_fxaa(vec3 color, vec2 uv_interp, vec2 pixel_size) {
 						  dir * rcpDirMin)) *
 			pixel_size;
 
-	vec3 rgbA = 0.5 * (texture2DLod(source, uv_interp + dir * (1.0 / 3.0 - 0.5), 0.0).xyz + texture2DLod(source, uv_interp + dir * (2.0 / 3.0 - 0.5), 0.0).xyz);
-	vec3 rgbB = rgbA * 0.5 + 0.25 * (texture2DLod(source, uv_interp + dir * -0.5, 0.0).xyz + texture2DLod(source, uv_interp + dir * 0.5, 0.0).xyz);
+	vec4 rgbA = 0.5 * (texture2DLod(source, uv_interp + dir * (1.0 / 3.0 - 0.5), 0.0) + texture2DLod(source, uv_interp + dir * (2.0 / 3.0 - 0.5), 0.0));
+	vec4 rgbB = rgbA * 0.5 + 0.25 * (texture2DLod(source, uv_interp + dir * -0.5, 0.0) + texture2DLod(source, uv_interp + dir * 0.5, 0.0));
 
-	float lumaB = dot(rgbB, luma);
-	if ((lumaB < lumaMin) || (lumaB > lumaMax)) {
-		return rgbA;
-	} else {
-		return rgbB;
+	float lumaB = dot(rgbB.rgb, luma) - ((1 - rgbB.a) / 8.0);
+	vec4 color_output = ((lumaB < lumaMin) || (lumaB > lumaMax)) ? rgbA : rgbB;
+	if (color_output.a == 0.0) {
+		color_output.rgb = vec3(0.0);
+	} else if (color_output.a < 1.0) {
+		color_output.rgb /= color_output.a;
 	}
+
+  return color_output;
 }
 
 void main() {
-	vec3 color = texture2DLod(source, uv_interp, 0.0).rgb;
+  vec4 color = texture2DLod(source, uv_interp, 0.0);
 
 #ifdef USE_FXAA
 	color = apply_fxaa(color, uv_interp, pixel_size);
@@ -336,12 +350,12 @@ void main() {
 	// Additional effects
 
 #ifdef USE_BCS
-	color = apply_bcs(color, bcs);
+  color.rgb = apply_bcs(color.rgb, bcs);
 #endif
 
 #ifdef USE_COLOR_CORRECTION
-	color = apply_color_correction(color, color_correction);
+  color.rgb = apply_color_correction(color.rgb, color_correction);
 #endif
 
-	gl_FragColor = vec4(color, 1.0);
+  gl_FragColor = color;
 }