Fix wrong face culling once and for all (#1277)

* Viewport swizzle support on NV and clip origin

* Initialize default viewport swizzle state, emulate viewport swizzle on shaders when not supported

* Address PR feedback

Ryujinx.Graphics.GAL/Capabilities.cs

@@ -5,26 +5,28 @@ namespace Ryujinx.Graphics.GAL
         public bool SupportsAstcCompression          { get; }
         public bool SupportsImageLoadFormatted       { get; }
         public bool SupportsNonConstantTextureOffset { get; }
+        public bool SupportsViewportSwizzle          { get; }
 
         public int   MaximumComputeSharedMemorySize { get; }
+        public float MaximumSupportedAnisotropy     { get; }
         public int   StorageBufferOffsetAlignment   { get; }
 
-        public float MaxSupportedAnisotropy { get; }
-
         public Capabilities(
             bool  supportsAstcCompression,
             bool  supportsImageLoadFormatted,
             bool  supportsNonConstantTextureOffset,
+            bool  supportsViewportSwizzle,
             int   maximumComputeSharedMemorySize,
-            int   storageBufferOffsetAlignment,
+            float maximumSupportedAnisotropy,
-            float maxSupportedAnisotropy)
+            int   storageBufferOffsetAlignment)
         {
             SupportsAstcCompression          = supportsAstcCompression;
             SupportsImageLoadFormatted       = supportsImageLoadFormatted;
             SupportsNonConstantTextureOffset = supportsNonConstantTextureOffset;
+            SupportsViewportSwizzle          = supportsViewportSwizzle;
             MaximumComputeSharedMemorySize   = maximumComputeSharedMemorySize;
+            MaximumSupportedAnisotropy       = maximumSupportedAnisotropy;
             StorageBufferOffsetAlignment     = storageBufferOffsetAlignment;
-            MaxSupportedAnisotropy           = maxSupportedAnisotropy;
         }
     }
 }

Ryujinx.Graphics.GAL/IPipeline.cs

@@ -42,6 +42,8 @@ namespace Ryujinx.Graphics.GAL
 
         void SetImage(int index, ShaderStage stage, ITexture texture);
 
+        void SetOrigin(Origin origin);
+
         void SetPointSize(float size);
 
         void SetPrimitiveRestart(bool enable, int index);

Ryujinx.Graphics.GAL/Origin.cs

@@ -0,0 +1,8 @@
+namespace Ryujinx.Graphics.GAL
+{
+    public enum Origin
+    {
+        UpperLeft,
+        LowerLeft
+    }
+}

Ryujinx.Graphics.GAL/ViewportSwizzle.cs

@@ -2,13 +2,15 @@ namespace Ryujinx.Graphics.GAL
 {
     public enum ViewportSwizzle
     {
-        PositiveX,
+        PositiveX = 0,
-        NegativeX,
+        NegativeX = 1,
-        PositiveY,
+        PositiveY = 2,
-        NegativeY,
+        NegativeY = 3,
-        PositiveZ,
+        PositiveZ = 4,
-        NegativeZ,
+        NegativeZ = 5,
-        PositiveW,
+        PositiveW = 6,
-        NegativeW
+        NegativeW = 7,
+
+        NegativeFlag = 1
     }
 }

Ryujinx.Graphics.Gpu/Engine/Methods.cs

@@ -422,10 +422,23 @@ namespace Ryujinx.Graphics.Gpu.Engine
 
             _context.Renderer.Pipeline.SetDepthMode(depthMode);
 
-            bool flipY = (state.Get<YControl>(MethodOffset.YControl) & YControl.NegateY) != 0;
+            YControl yControl = state.Get<YControl>(MethodOffset.YControl);
-            float yFlip = flipY ? -1 : 1;
 
-            Viewport[] viewports = new Viewport[Constants.TotalViewports];
+            bool   flipY  = yControl.HasFlag(YControl.NegateY);
+            Origin origin = yControl.HasFlag(YControl.TriangleRastFlip) ? Origin.LowerLeft : Origin.UpperLeft;
+            
+            _context.Renderer.Pipeline.SetOrigin(origin);
+
+            // The triangle rast flip flag only affects rasterization, the viewport is not flipped.
+            // Setting the origin mode to upper left on the host, however, not onlyy affects rasterization,
+            // but also flips the viewport.
+            // We negate the effects of flipping the viewport by flipping it again using the viewport swizzle.
+            if (origin == Origin.UpperLeft)
+            {
+                flipY = !flipY;
+            }
+
+            Span<Viewport> viewports = stackalloc Viewport[Constants.TotalViewports];
 
             for (int index = 0; index < Constants.TotalViewports; index++)
             {
@@ -435,17 +448,42 @@ namespace Ryujinx.Graphics.Gpu.Engine
                 float x = transform.TranslateX - MathF.Abs(transform.ScaleX);
                 float y = transform.TranslateY - MathF.Abs(transform.ScaleY);
 
-                float width  = transform.ScaleX * 2;
+                float width  = MathF.Abs(transform.ScaleX) * 2;
-                float height = transform.ScaleY * 2 * yFlip;
+                float height = MathF.Abs(transform.ScaleY) * 2;
 
                 RectangleF region = new RectangleF(x, y, width, height);
 
+                ViewportSwizzle swizzleX = transform.UnpackSwizzleX();
+                ViewportSwizzle swizzleY = transform.UnpackSwizzleY();
+                ViewportSwizzle swizzleZ = transform.UnpackSwizzleZ();
+                ViewportSwizzle swizzleW = transform.UnpackSwizzleW();
+
+                if (transform.ScaleX < 0)
+                {
+                    swizzleX ^= ViewportSwizzle.NegativeFlag;
+                }
+
+                if (flipY)
+                {
+                    swizzleY ^= ViewportSwizzle.NegativeFlag;
+                }
+
+                if (transform.ScaleY < 0)
+                {
+                    swizzleY ^= ViewportSwizzle.NegativeFlag;
+                }
+
+                if (transform.ScaleZ < 0)
+                {
+                    swizzleZ ^= ViewportSwizzle.NegativeFlag;
+                }
+
                 viewports[index] = new Viewport(
                     region,
-                    transform.UnpackSwizzleX(),
+                    swizzleX,
-                    transform.UnpackSwizzleY(),
+                    swizzleY,
-                    transform.UnpackSwizzleZ(),
+                    swizzleZ,
-                    transform.UnpackSwizzleW(),
+                    swizzleW,
                     extents.DepthNear,
                     extents.DepthFar);
             }

Ryujinx.Graphics.Gpu/Image/Sampler.cs

@@ -41,7 +41,7 @@ namespace Ryujinx.Graphics.Gpu.Image
             float mipLodBias = descriptor.UnpackMipLodBias();
 
             float maxRequestedAnisotropy = GraphicsConfig.MaxAnisotropy >= 0 && GraphicsConfig.MaxAnisotropy <= 16 ? GraphicsConfig.MaxAnisotropy : descriptor.UnpackMaxAnisotropy();
-            float maxSupportedAnisotropy = context.Capabilities.MaxSupportedAnisotropy;
+            float maxSupportedAnisotropy = context.Capabilities.MaximumSupportedAnisotropy;
 
             if (maxRequestedAnisotropy > maxSupportedAnisotropy)
                 maxRequestedAnisotropy = maxSupportedAnisotropy;

Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs

@@ -3,6 +3,7 @@ using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Gpu.Image;
 using Ryujinx.Graphics.Gpu.State;
 using Ryujinx.Graphics.Shader;
+using System;
 
 namespace Ryujinx.Graphics.Gpu.Shader
 {
@@ -187,6 +188,12 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// <returns>True if the GPU and driver supports non-constant texture offsets, false otherwise</returns>
         public bool QuerySupportsNonConstantTextureOffset() => _context.Capabilities.SupportsNonConstantTextureOffset;
 
+        /// <summary>
+        /// Queries host GPU viewport swizzle support.
+        /// </summary>
+        /// <returns>True if the GPU and driver supports viewport swizzle, false otherwise</returns>
+        public bool QuerySupportsViewportSwizzle() => _context.Capabilities.SupportsViewportSwizzle;
+
         /// <summary>
         /// Queries texture format information, for shaders using image load or store.
         /// </summary>
@@ -250,6 +257,24 @@ namespace Ryujinx.Graphics.Gpu.Shader
             };
         }
 
+        public int QueryViewportSwizzle(int component)
+        {
+            YControl yControl = _state.Get<YControl>(MethodOffset.YControl);
+
+            bool flipY = yControl.HasFlag(YControl.NegateY) ^ !yControl.HasFlag(YControl.TriangleRastFlip);
+
+            ViewportTransform transform = _state.Get<ViewportTransform>(MethodOffset.ViewportTransform, 0);
+
+            return component switch
+            {
+                0 => (int)(transform.UnpackSwizzleX() ^ (transform.ScaleX < 0 ? ViewportSwizzle.NegativeFlag : 0)),
+                1 => (int)(transform.UnpackSwizzleY() ^ (transform.ScaleY < 0 ? ViewportSwizzle.NegativeFlag : 0) ^ (flipY ? ViewportSwizzle.NegativeFlag : 0)),
+                2 => (int)(transform.UnpackSwizzleZ() ^ (transform.ScaleZ < 0 ? ViewportSwizzle.NegativeFlag : 0)),
+                3 => (int)transform.UnpackSwizzleW(),
+                _ => throw new ArgumentOutOfRangeException(nameof(component))
+            };
+        }
+
         /// <summary>
         /// Gets the texture descriptor for a given texture on the pool.
         /// </summary>

Ryujinx.Graphics.Gpu/State/GpuState.cs

@@ -146,6 +146,9 @@ namespace Ryujinx.Graphics.Gpu.State
             {
                 memory[(int)MethodOffset.ViewportExtents + index * 4 + 2] = 0;
                 memory[(int)MethodOffset.ViewportExtents + index * 4 + 3] = 0x3F800000;
+
+                // Set swizzle to +XYZW
+                memory[(int)MethodOffset.ViewportTransform + index * 8 + 6] = 0x6420;
             }
 
             // Viewport transform enable.

Ryujinx.Graphics.OpenGL/EnumConversion.cs

@@ -416,5 +416,32 @@ namespace Ryujinx.Graphics.OpenGL
 
             return TextureTarget.Texture2D;
         }
+
+        public static NvViewportSwizzle Convert(this ViewportSwizzle swizzle)
+        {
+            switch (swizzle)
+            {
+                case ViewportSwizzle.PositiveX:
+                    return NvViewportSwizzle.ViewportSwizzlePositiveXNv;
+                case ViewportSwizzle.PositiveY:
+                    return NvViewportSwizzle.ViewportSwizzlePositiveYNv;
+                case ViewportSwizzle.PositiveZ:
+                    return NvViewportSwizzle.ViewportSwizzlePositiveZNv;
+                case ViewportSwizzle.PositiveW:
+                    return NvViewportSwizzle.ViewportSwizzlePositiveWNv;
+                case ViewportSwizzle.NegativeX:
+                    return NvViewportSwizzle.ViewportSwizzleNegativeXNv;
+                case ViewportSwizzle.NegativeY:
+                    return NvViewportSwizzle.ViewportSwizzleNegativeYNv;
+                case ViewportSwizzle.NegativeZ:
+                    return NvViewportSwizzle.ViewportSwizzleNegativeZNv;
+                case ViewportSwizzle.NegativeW:
+                    return NvViewportSwizzle.ViewportSwizzleNegativeWNv;
+            }
+
+            Logger.PrintDebug(LogClass.Gpu, $"Invalid {nameof(ViewportSwizzle)} enum value: {swizzle}.");
+
+            return NvViewportSwizzle.ViewportSwizzlePositiveXNv;
+        }
     }
 }

Ryujinx.Graphics.OpenGL/HwCapabilities.cs

@@ -7,6 +7,7 @@ namespace Ryujinx.Graphics.OpenGL
     {
         private static readonly Lazy<bool> _supportsAstcCompression    = new Lazy<bool>(() => HasExtension("GL_KHR_texture_compression_astc_ldr"));
         private static readonly Lazy<bool> _supportsImageLoadFormatted = new Lazy<bool>(() => HasExtension("GL_EXT_shader_image_load_formatted"));
+        private static readonly Lazy<bool> _supportsViewportSwizzle    = new Lazy<bool>(() => HasExtension("GL_NV_viewport_swizzle"));
 
         private static readonly Lazy<int> _maximumComputeSharedMemorySize = new Lazy<int>(() => GetLimit(All.MaxComputeSharedMemorySize));
         private static readonly Lazy<int> _storageBufferOffsetAlignment   = new Lazy<int>(() => GetLimit(All.ShaderStorageBufferOffsetAlignment));
@@ -27,12 +28,13 @@ namespace Ryujinx.Graphics.OpenGL
 
         public static bool SupportsAstcCompression          => _supportsAstcCompression.Value;
         public static bool SupportsImageLoadFormatted       => _supportsImageLoadFormatted.Value;
+        public static bool SupportsViewportSwizzle          => _supportsViewportSwizzle.Value;
         public static bool SupportsNonConstantTextureOffset => _gpuVendor.Value == GpuVendor.Nvidia;
 
         public static int MaximumComputeSharedMemorySize => _maximumComputeSharedMemorySize.Value;
         public static int StorageBufferOffsetAlignment   => _storageBufferOffsetAlignment.Value;
 
-        public static float MaxSupportedAnisotropy => _maxSupportedAnisotropy.Value;
+        public static float MaximumSupportedAnisotropy => _maxSupportedAnisotropy.Value;
 
         private static bool HasExtension(string name)
         {

Ryujinx.Graphics.OpenGL/Pipeline.cs

@@ -650,6 +650,13 @@ namespace Ryujinx.Graphics.OpenGL
             _vertexArray.SetIndexBuffer(buffer.Handle);
         }
 
+        public void SetOrigin(Origin origin)
+        {
+            ClipOrigin clipOrigin = origin == Origin.UpperLeft ? ClipOrigin.UpperLeft : ClipOrigin.LowerLeft;
+
+            SetOrigin(clipOrigin);
+        }
+
         public void SetPointSize(float size)
         {
             GL.PointSize(size);
@@ -854,8 +861,6 @@ namespace Ryujinx.Graphics.OpenGL
 
         public void SetViewports(int first, ReadOnlySpan<Viewport> viewports)
         {
-            bool flipY = false;
-
             float[] viewportArray = new float[viewports.Length * 4];
 
             double[] depthRangeArray = new double[viewports.Length * 2];
@@ -869,17 +874,14 @@ namespace Ryujinx.Graphics.OpenGL
                 viewportArray[viewportElemIndex + 0] = viewport.Region.X;
                 viewportArray[viewportElemIndex + 1] = viewport.Region.Y;
 
-                // OpenGL does not support per-viewport flipping, so
+                if (HwCapabilities.SupportsViewportSwizzle)
-                // instead we decide that based on the viewport 0 value.
-                // It will apply to all viewports.
-                if (index == 0)
                 {
-                    flipY = viewport.Region.Height < 0;
+                    GL.NV.ViewportSwizzle(
-                }
+                        index,
-
+                        viewport.SwizzleX.Convert(),
-                if (viewport.SwizzleY == ViewportSwizzle.NegativeY)
+                        viewport.SwizzleY.Convert(),
-                {
+                        viewport.SwizzleZ.Convert(),
-                    flipY = !flipY;
+                        viewport.SwizzleW.Convert());
                 }
 
                 viewportArray[viewportElemIndex + 2] = MathF.Abs(viewport.Region.Width);
@@ -892,8 +894,6 @@ namespace Ryujinx.Graphics.OpenGL
             GL.ViewportArray(first, viewports.Length, viewportArray);
 
             GL.DepthRangeArray(first, viewports.Length, depthRangeArray);
-
-            SetOrigin(flipY ? ClipOrigin.UpperLeft : ClipOrigin.LowerLeft);
         }
 
         public void TextureBarrier()

Ryujinx.Graphics.OpenGL/Renderer.cs

@@ -75,9 +75,10 @@ namespace Ryujinx.Graphics.OpenGL
                 HwCapabilities.SupportsAstcCompression,
                 HwCapabilities.SupportsImageLoadFormatted,
                 HwCapabilities.SupportsNonConstantTextureOffset,
+                HwCapabilities.SupportsViewportSwizzle,
                 HwCapabilities.MaximumComputeSharedMemorySize,
-                HwCapabilities.StorageBufferOffsetAlignment,
+                HwCapabilities.MaximumSupportedAnisotropy,
-                HwCapabilities.MaxSupportedAnisotropy);
+                HwCapabilities.StorageBufferOffsetAlignment);
         }
 
         public void SetBufferData(BufferHandle buffer, int offset, ReadOnlySpan<byte> data)

Ryujinx.Graphics.Shader/IGpuAccessor.cs

@@ -64,9 +64,22 @@
             return true;
         }
 
+        public bool QuerySupportsViewportSwizzle()
+        {
+            return true;
+        }
+
         public TextureFormat QueryTextureFormat(int handle)
         {
             return TextureFormat.R8G8B8A8Unorm;
         }
+
+        public int QueryViewportSwizzle(int component)
+        {
+            // Bit 0: Negate flag.
+            // Bits 2-1: Component.
+            // Example: 0b110 = W, 0b111 = -W, 0b000 = X, 0b010 = Y etc.
+            return component << 1;
+        }
     }
 }

Ryujinx.Graphics.Shader/Translation/EmitterContext.cs

@@ -11,9 +11,7 @@ namespace Ryujinx.Graphics.Shader.Translation
         public Block  CurrBlock { get; set; }
         public OpCode CurrOp    { get; set; }
 
-        private ShaderConfig _config;
+        public ShaderConfig Config { get; }
-
-        public ShaderConfig Config => _config;
 
         private List<Operation> _operations;
 
@@ -21,7 +19,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public EmitterContext(ShaderConfig config)
         {
-            _config = config;
+            Config = config;
 
             _operations = new List<Operation>();
 
@@ -61,13 +59,40 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public void PrepareForReturn()
         {
-            if (_config.Stage == ShaderStage.Fragment)
+            if (Config.Stage == ShaderStage.Vertex && (Config.Flags & TranslationFlags.VertexA) == 0)
             {
-                if (_config.OmapDepth)
+                // Here we attempt to implement viewport swizzle on the vertex shader.
+                // Perform permutation and negation of the output gl_Position components.
+                // Note that per-viewport swizzling can't be supported using this approach.
+                int swizzleX = Config.GpuAccessor.QueryViewportSwizzle(0);
+                int swizzleY = Config.GpuAccessor.QueryViewportSwizzle(1);
+                int swizzleZ = Config.GpuAccessor.QueryViewportSwizzle(2);
+                int swizzleW = Config.GpuAccessor.QueryViewportSwizzle(3);
+
+                bool nonStandardSwizzle = swizzleX != 0 || swizzleY != 2 || swizzleZ != 4 || swizzleW != 6;
+
+                if (!Config.GpuAccessor.QuerySupportsViewportSwizzle() && nonStandardSwizzle)
+                {
+                    Operand[] temp = new Operand[4];
+
+                    temp[0] = this.Copy(Attribute(AttributeConsts.PositionX));
+                    temp[1] = this.Copy(Attribute(AttributeConsts.PositionY));
+                    temp[2] = this.Copy(Attribute(AttributeConsts.PositionZ));
+                    temp[3] = this.Copy(Attribute(AttributeConsts.PositionW));
+
+                    this.Copy(Attribute(AttributeConsts.PositionX), this.FPNegate(temp[(swizzleX >> 1) & 3], (swizzleX & 1) != 0));
+                    this.Copy(Attribute(AttributeConsts.PositionY), this.FPNegate(temp[(swizzleY >> 1) & 3], (swizzleY & 1) != 0));
+                    this.Copy(Attribute(AttributeConsts.PositionZ), this.FPNegate(temp[(swizzleZ >> 1) & 3], (swizzleZ & 1) != 0));
+                    this.Copy(Attribute(AttributeConsts.PositionW), this.FPNegate(temp[(swizzleW >> 1) & 3], (swizzleW & 1) != 0));
+                }
+            }
+            else if (Config.Stage == ShaderStage.Fragment)
+            {
+                if (Config.OmapDepth)
                 {
                     Operand dest = Attribute(AttributeConsts.FragmentOutputDepth);
 
-                    Operand src = Register(_config.GetDepthRegister(), RegisterType.Gpr);
+                    Operand src = Register(Config.GetDepthRegister(), RegisterType.Gpr);
 
                     this.Copy(dest, src);
                 }
@@ -76,7 +101,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
                 for (int attachment = 0; attachment < 8; attachment++)
                 {
-                    OmapTarget target = _config.OmapTargets[attachment];
+                    OmapTarget target = Config.OmapTargets[attachment];
 
                     for (int component = 0; component < 4; component++)
                     {

Ryujinx.Graphics.Shader/Translation/TranslationFlags.cs

@@ -7,7 +7,8 @@ namespace Ryujinx.Graphics.Shader.Translation
     {
         None = 0,
 
-        Compute   = 1 << 0,
+        VertexA   = 1 << 0,
-        DebugMode = 1 << 1
+        Compute   = 1 << 1,
+        DebugMode = 1 << 2
     }
 }

Ryujinx.Graphics.Shader/Translation/Translator.cs

@@ -23,7 +23,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public static ShaderProgram Translate(ulong addressA, ulong addressB, IGpuAccessor gpuAccessor, TranslationFlags flags)
         {
-            Operation[] opsA = DecodeShader(addressA, gpuAccessor, flags, out _, out int sizeA);
+            Operation[] opsA = DecodeShader(addressA, gpuAccessor, flags | TranslationFlags.VertexA, out _, out int sizeA);
             Operation[] opsB = DecodeShader(addressB, gpuAccessor, flags, out ShaderConfig config, out int sizeB);
 
             return Translate(Combine(opsA, opsB), config, sizeB, sizeA);