naomi2: fix modifier volumes clipping

Don't clip modifier volumes but tesselate triangles intersecting the
near plane. Then project clipped vertices onto it in the vertex shader.
Issue flyinghead#1651

core/hw/pvr/elan.cpp

@@ -983,7 +983,7 @@ class ModifierVolumeClipper
 				tri.x2 * curMatrix[0][2] + tri.y2 * curMatrix[1][2] + tri.z2 * curMatrix[2][2] + curMatrix[3][2]
 			};
 			dist = -dist - nearPlane;
-			ModTriangle newTri;
+			ModTriangle newTri[2];
 			int n = sutherlandHodgmanClip(dist, tri, newTri);
 			switch (n)
 			{
@@ -993,9 +993,10 @@ class ModifierVolumeClipper
 			case 3:
 				ta_add_triangle(tri);
 				break;
-			case 4:
+			case 5:
 				ta_add_triangle(tri);
-				ta_add_triangle(newTri);
+				ta_add_triangle(newTri[0]);
+				ta_add_triangle(newTri[1]);
 				break;
 			}
 		}
@@ -1016,86 +1017,78 @@ class ModifierVolumeClipper
 	}
 
 	// Clip the triangle 'trig' with respect to the provided distances to the clipping plane.
-	int sutherlandHodgmanClip(glm::vec3& dist, ModTriangle& trig, ModTriangle& newTrig)
+	int sutherlandHodgmanClip(glm::vec3& dist, ModTriangle& trig, ModTriangle *newTrig)
 	{
 		constexpr float clipEpsilon = 0.f; //0.00001;
 		constexpr float clipEpsilon2 = 0.f; //0.01;
 
-		if (!glm::any(glm::greaterThanEqual(dist , glm::vec3(clipEpsilon2))))
-			// all clipped
-			return 0;
+		if (!glm::any(glm::greaterThanEqual(dist , glm::vec3(clipEpsilon2)))) {
+			// all clipped: leave it alone as it will be projected onto the near plane in the shader
+			return 3;
+		}
 		if (glm::all(glm::greaterThanEqual(dist , glm::vec3(-clipEpsilon))))
 			// none clipped
 			return 3;
 
-		// There are either 1 or 2 vertices above the clipping plane.
+		// There are either 1 or 2 vertices above the clipping plane. Tesselate into 3 triangles along the plane.
 		glm::bvec3 above = glm::greaterThanEqual(dist, glm::vec3(0.f));
-		bool nextIsAbove;
 		glm::vec3 v0(trig.x0, trig.y0, trig.z0);
 		glm::vec3 v1(trig.x1, trig.y1, trig.z1);
 		glm::vec3 v2(trig.x2, trig.y2, trig.z2);
-		glm::vec3 v3;
-		// Find the CCW-most vertex above the plane.
-		if (above[1] && !above[0])
+		glm::vec3 v3, v4;
+		// Find the lonely vertex on one side of the plane
+		if (above[0] == above[2])
 		{
-			// Cycle once CCW. Use v3 as a temp
-			nextIsAbove = above[2];
+			// this is vertex 1 so cycle CCW
 			v3 = v0;
 			v0 = v1;
 			v1 = v2;
 			v2 = v3;
 			dist = glm::vec3(dist.y, dist.z, dist.x);
 		}
-		else if (above[2] && !above[1])
+		else if (above[0] == above[1])
 		{
-			// Cycle once CW. Use v3 as a temp.
-			nextIsAbove = above[0];
+			// this is vertex 2 so cycle CW
 			v3 = v2;
 			v2 = v1;
 			v1 = v0;
 			v0 = v3;
 			dist = glm::vec3(dist.z, dist.x, dist.y);
 		}
-		else
-			nextIsAbove = above[1];
+		v3 = intersect(v0, dist[0], v1, dist[1]);
+		v4 = intersect(v0, dist[0], v2, dist[2]);
+		// v0 v3 v4
 		trig.x0 = v0.x;
 		trig.y0 = v0.y;
 		trig.z0 = v0.z;
-		// We always need to clip v2-v0.
-		v3 = intersect(v0, dist[0], v2, dist[2]);
-		if (nextIsAbove)
-		{
-			v2 = intersect(v1, dist[1], v2, dist[2]);
-			trig.x1 = v1.x;
-			trig.y1 = v1.y;
-			trig.z1 = v1.z;
-			trig.x2 = v2.x;
-			trig.y2 = v2.y;
-			trig.z2 = v2.z;
-			newTrig.x0 = v0.x;
-			newTrig.y0 = v0.y;
-			newTrig.z0 = v0.z;
-			newTrig.x1 = v2.x;
-			newTrig.y1 = v2.y;
-			newTrig.z1 = v2.z;
-			newTrig.x2 = v3.x;
-			newTrig.y2 = v3.y;
-			newTrig.z2 = v3.z;
-
-			return 4;
-		}
-		else
-		{
-			v1 = intersect(v0, dist[0], v1, dist[1]);
-			trig.x1 = v1.x;
-			trig.y1 = v1.y;
-			trig.z1 = v1.z;
-			trig.x2 = v3.x;
-			trig.y2 = v3.y;
-			trig.z2 = v3.z;
-
-			return 3;
-		}
+		trig.x1 = v3.x;
+		trig.y1 = v3.y;
+		trig.z1 = v3.z;
+		trig.x2 = v4.x;
+		trig.y2 = v4.y;
+		trig.z2 = v4.z;
+		// v3 v1 v4
+		newTrig[0].x0 = v3.x;
+		newTrig[0].y0 = v3.y;
+		newTrig[0].z0 = v3.z;
+		newTrig[0].x1 = v1.x;
+		newTrig[0].y1 = v1.y;
+		newTrig[0].z1 = v1.z;
+		newTrig[0].x2 = v4.x;
+		newTrig[0].y2 = v4.y;
+		newTrig[0].z2 = v4.z;
+		// v2 v4 v1
+		newTrig[1].x0 = v2.x;
+		newTrig[1].y0 = v2.y;
+		newTrig[1].z0 = v2.z;
+		newTrig[1].x1 = v4.x;
+		newTrig[1].y1 = v4.y;
+		newTrig[1].z1 = v4.z;
+		newTrig[1].x2 = v1.x;
+		newTrig[1].y2 = v1.y;
+		newTrig[1].z2 = v1.z;
+
+		return 5;
 	}
 
 	bool enabled;
@@ -1327,15 +1320,15 @@ static void sendPolygon(ICHList *list)
 	case ICHList::VTX_TYPE_V:
 		{
 			N2_VERTEX *vtx = (N2_VERTEX *)((u8 *)list + sizeof(ICHList));
-			if (!isBetweenNearAndFar(vtx, list->vtxCount, needClipping))
-				break;
 			int listType = ta_get_list_type();
 			if (listType == -1)
 				listType = list->pcw.listType;
 			if (listType & 1)
-				sendMVPolygon(list, vtx, needClipping);
+				sendMVPolygon(list, vtx, true);
 			else
 			{
+				if (!isBetweenNearAndFar(vtx, list->vtxCount, needClipping))
+					break;
 				PolyParam pp{};
 				pp.pcw.Shadow = list->pcw.shadow;
 				pp.pcw.Texture = 0;
@@ -1356,15 +1349,15 @@ static void sendPolygon(ICHList *list)
 	case ICHList::VTX_TYPE_VU:
 		{
 			N2_VERTEX_VU *vtx = (N2_VERTEX_VU *)((u8 *)list + sizeof(ICHList));
-			if (!isBetweenNearAndFar(vtx, list->vtxCount, needClipping))
-				break;
 			int listType = ta_get_list_type();
 			if (listType == -1)
 				listType = list->pcw.listType;
 			if (listType  & 1)
-				sendMVPolygon(list, vtx, needClipping);
+				sendMVPolygon(list, vtx, true);
 			else
 			{
+				if (!isBetweenNearAndFar(vtx, list->vtxCount, needClipping))
+					break;
 				PolyParam pp{};
 				pp.pcw.Shadow = list->pcw.shadow;
 				pp.pcw.Texture = list->pcw.texture;

core/rend/dx11/dx11_naomi2.cpp

@@ -131,7 +131,9 @@ VertexOut main(in VertexIn vin)
 	#endif
 	vo.index = uint(polyNumber) + vin.vertexId;
 #endif
-
+#if MODIFIER_VOLUME == 1
+	vo.pos.z = min(vo.pos.z, -0.001f);
+#endif
 	vo.pos = mul(projMat, vo.pos);
 
 	vo.pos = float4(vo.pos.xy / vo.pos.w, 1.f / vo.pos.w, 1.f);

core/rend/dx11/dx11_shaders.cpp

@@ -460,6 +460,7 @@ enum VertexMacroEnum {
 	MacroPositionOnly,
 	MacroTwoVolumes,
 	MacroLightOn,
+	MacroModifierVolume,
 };
 
 static D3D_SHADER_MACRO VertexMacros[]
@@ -469,6 +470,7 @@ static D3D_SHADER_MACRO VertexMacros[]
 	{ "POSITION_ONLY", "0" },
 	{ "pp_TwoVolumes", "0" },
 	{ "LIGHT_ON", "1" },
+	{ "MODIFIER_VOLUME", "0" },
 	{ nullptr, nullptr }
 };
 
@@ -573,6 +575,7 @@ const ComPtr<ID3D11VertexShader>& DX11Shaders::getVertexShader(bool gouraud, boo
 			VertexMacros[MacroPositionOnly].Definition = MacroValues[false];
 			VertexMacros[MacroTwoVolumes].Definition = MacroValues[false];
 			VertexMacros[MacroLightOn].Definition = MacroValues[true];
+			VertexMacros[MacroModifierVolume].Definition = MacroValues[false];
 			std::string source(DX11N2VertexShader);
 			source += std::string("\n") + DX11N2ColorShader;
 			vertexShader = compileVS(source.c_str(), "main", VertexMacros);
@@ -599,6 +602,7 @@ const ComPtr<ID3D11VertexShader>& DX11Shaders::getMVVertexShader(bool naomi2)
 			VertexMacros[MacroPositionOnly].Definition = MacroValues[true];
 			VertexMacros[MacroTwoVolumes].Definition = MacroValues[false];
 			VertexMacros[MacroLightOn].Definition = MacroValues[false];
+			VertexMacros[MacroModifierVolume].Definition = MacroValues[true];
 			modVolVertexShaders[index] = compileVS(DX11N2VertexShader, "main", VertexMacros);
 		}
 	}
@@ -689,6 +693,7 @@ ComPtr<ID3DBlob> DX11Shaders::getVertexShaderBlob()
 	// FIXME code dup
 	VertexMacros[MacroPositionOnly].Definition = MacroValues[false];
 	VertexMacros[MacroTwoVolumes].Definition = MacroValues[false];
+	VertexMacros[MacroModifierVolume].Definition = MacroValues[false];
 	std::string source(DX11N2VertexShader);
 	source += std::string("\n") + DX11N2ColorShader;
 	return compileShader(source.c_str(), "main", "vs_4_0", VertexMacros);
@@ -700,6 +705,7 @@ ComPtr<ID3DBlob> DX11Shaders::getMVVertexShaderBlob()
 	VertexMacros[MacroGouraud].Definition = MacroValues[false];
 	VertexMacros[MacroPositionOnly].Definition = MacroValues[true];
 	VertexMacros[MacroTwoVolumes].Definition = MacroValues[false];
+	VertexMacros[MacroModifierVolume].Definition = MacroValues[true];
 	return compileShader(DX11N2VertexShader, "main", "vs_4_0", VertexMacros);
 }
 

core/rend/dx11/dx11context_lr.h

@@ -89,7 +89,6 @@ class DX11Context : public GraphicsContext
 	int renderTargetHeight = 0;
 	BlendStates blendStates;
 	std::unique_ptr<Quad> quad;
-	ComPtr<ID3D11RasterizerState> rasterizerState;
 
 	D3D_FEATURE_LEVEL featureLevel{};
 	bool supportedTexFormats[5] {}; // indexed by TextureType enum

core/rend/dx11/oit/dx11_oitshaders.cpp

@@ -781,6 +781,7 @@ enum VertexMacroEnum {
 	MacroDivPosZ,
 	MacroPositionOnly,
 	MacroLightOn,
+	MacroModifierVolume,
 };
 
 static D3D_SHADER_MACRO VertexMacros[]
@@ -790,6 +791,7 @@ static D3D_SHADER_MACRO VertexMacros[]
 	{ "DIV_POS_Z", "0" },
 	{ "POSITION_ONLY", "0" },
 	{ "LIGHT_ON", "1" },
+	{ "MODIFIER_VOLUME", "0" },
 	{ nullptr, nullptr }
 };
 
@@ -900,6 +902,7 @@ const ComPtr<ID3D11VertexShader>& DX11OITShaders::getVertexShader(bool gouraud,
 			VertexMacros[MacroPositionOnly].Definition = MacroValues[positionOnly];
 			VertexMacros[MacroTwoVolumes].Definition = MacroValues[twoVolumes];
 			VertexMacros[MacroLightOn].Definition = MacroValues[lightOn];
+			VertexMacros[MacroModifierVolume].Definition = MacroValues[false];
 			std::string source(DX11N2VertexShader);
 			if (!positionOnly && lightOn)
 				source += std::string("\n") + DX11N2ColorShader;
@@ -927,6 +930,7 @@ const ComPtr<ID3D11VertexShader>& DX11OITShaders::getMVVertexShader(bool naomi2)
 			VertexMacros[MacroPositionOnly].Definition = MacroValues[true];
 			VertexMacros[MacroTwoVolumes].Definition = MacroValues[false];
 			VertexMacros[MacroLightOn].Definition = MacroValues[false];
+			VertexMacros[MacroModifierVolume].Definition = MacroValues[true];
 			mvVertexShader = compileVS(DX11N2VertexShader, "main", VertexMacros);
 		}
 	}
@@ -1058,6 +1062,7 @@ ComPtr<ID3DBlob> DX11OITShaders::getVertexShaderBlob()
 	// FIXME code dup
 	VertexMacros[MacroPositionOnly].Definition = MacroValues[false];
 	VertexMacros[MacroTwoVolumes].Definition = MacroValues[true];
+	VertexMacros[MacroModifierVolume].Definition = MacroValues[false];
 	std::string source(DX11N2VertexShader);
 	source += std::string("\n") + DX11N2ColorShader;
 	return compileShader(source.c_str(), "main", "vs_5_0", VertexMacros);
@@ -1069,6 +1074,7 @@ ComPtr<ID3DBlob> DX11OITShaders::getMVVertexShaderBlob()
 	VertexMacros[MacroGouraud].Definition = MacroValues[false];
 	VertexMacros[MacroPositionOnly].Definition = MacroValues[true];
 	VertexMacros[MacroTwoVolumes].Definition = MacroValues[false];
+	VertexMacros[MacroModifierVolume].Definition = MacroValues[true];
 	return compileShader(DX11N2VertexShader, "main", "vs_5_0", VertexMacros);
 }
 

core/rend/gl4/gl4naomi2.cpp

@@ -34,6 +34,7 @@ N2Vertex4Source::N2Vertex4Source(const gl4PipelineShader* shader) : OpenGl4Sourc
 {
 	addConstant("OIT_RENDER");
 	addConstant("DIV_POS_Z", false);
+	addConstant("MODIFIER_VOLUME", 0);
 	if (shader == nullptr)
 	{
 		addConstant("POSITION_ONLY", 1);

core/rend/gl4/gles.cpp

@@ -642,6 +642,7 @@ static void create_modvol_shader()
 	gl4.modvol_shader.ndcMat = glGetUniformLocation(gl4.modvol_shader.program, "ndcMat");
 
 	N2Vertex4Source n2VertexShader;
+	n2VertexShader.setConstant("MODIFIER_VOLUME", true);
 	fragmentShader.setConstant("DIV_POS_Z", false);
 	gl4.n2ModVolShader.program = gl_CompileAndLink(n2VertexShader.generate().c_str(), fragmentShader.generate().c_str());
 	gl4.n2ModVolShader.ndcMat = glGetUniformLocation(gl4.n2ModVolShader.program, "ndcMat");

core/rend/gles/naomi2.cpp

@@ -121,6 +121,9 @@ void main()
 			computeEnvMap(vtx_uv.xy, vpos.xyz, vnorm);
 	#endif
 #endif
+#if MODIFIER_VOLUME == 1
+	vpos.z = min(vpos.z, -0.001);
+#endif
 
 	vpos = projMat * vpos;
 	wDivide(vpos);
@@ -348,17 +351,18 @@ void computeBumpMap(inout vec4 color0, vec4 color1, vec3 position, vec3 normal,
 
 )";
 
-N2VertexSource::N2VertexSource(bool gouraud, bool geometryOnly, bool texture) : OpenGlSource()
+N2VertexSource::N2VertexSource(bool gouraud, bool modifierVolume, bool texture) : OpenGlSource()
 {
-	addConstant("pp_Gouraud", gouraud);
-	addConstant("POSITION_ONLY", geometryOnly);
+	addConstant("pp_Gouraud", (int)gouraud);
+	addConstant("POSITION_ONLY", (int)modifierVolume);
 	addConstant("pp_TwoVolumes", 0);
 	addConstant("pp_Texture", (int)texture);
 	addConstant("LIGHT_ON", 1);
+	addConstant("MODIFIER_VOLUME", (int)modifierVolume);
 
 	addSource(VertexCompatShader);
 	addSource(GouraudSource);
-	if (!geometryOnly)
+	if (!modifierVolume)
 		addSource(N2ColorShader);
 	addSource(N2VertexShader);
 }

core/rend/gles/naomi2.h

@@ -22,7 +22,7 @@
 class N2VertexSource : public OpenGlSource
 {
 public:
-	N2VertexSource(bool gouraud, bool geometryOnly, bool texture);
+	N2VertexSource(bool gouraud, bool modifierVolume, bool texture);
 };
 
 template<typename ShaderType>

core/rend/vulkan/shaders.cpp

@@ -743,6 +743,7 @@ void wDivide(inout vec4 vpos)
 void main()
 {
 	vec4 vpos = n2Uniform.mvMat * in_pos;
+	vpos.z = min(vpos.z, -0.001);
 	vpos = n2Uniform.projMat * vpos;
 	wDivide(vpos);