LibHac/tests/LibHac.Tests/Util/CharacterEncodingTests.cs

// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using LibHac.Util;
using Xunit;

namespace LibHac.Tests.Util
{
    public class CharacterEncodingTests
    {
        // Most of these tests are stolen from .NET's UTF-8 tests. Some of the comments in this file may
        // mention code paths and functions being tested in the .NET runtime that don't apply here as a result.

        // ReSharper disable InconsistentNaming UnusedMember.Local
        private const string X_UTF8 = "58"; // U+0058 LATIN CAPITAL LETTER X, 1 byte
        private const string X_UTF16 = "X";
        
        private const string Y_UTF8 = "59"; // U+0058 LATIN CAPITAL LETTER Y, 1 byte
        private const string Y_UTF16 = "Y";

        private const string Z_UTF8 = "5A"; // U+0058 LATIN CAPITAL LETTER Z, 1 byte
        private const string Z_UTF16 = "Z";

        private const string E_ACUTE_UTF8 = "C3A9"; // U+00E9 LATIN SMALL LETTER E WITH ACUTE, 2 bytes
        private const string E_ACUTE_UTF16 = "\u00E9";

        private const string EURO_SYMBOL_UTF8 = "E282AC"; // U+20AC EURO SIGN, 3 bytes
        private const string EURO_SYMBOL_UTF16 = "\u20AC";

        private const string REPLACEMENT_CHAR_UTF8 = "EFBFBD"; // U+FFFD REPLACEMENT CHAR, 3 bytes
        private const string REPLACEMENT_CHAR_UTF16 = "\uFFFD";

        private const string GRINNING_FACE_UTF8 = "F09F9880"; // U+1F600 GRINNING FACE, 4 bytes
        private const string GRINNING_FACE_UTF16 = "\U0001F600";

        private const string WOMAN_CARTWHEELING_MEDSKIN_UTF16 = "\U0001F938\U0001F3FD\u200D\u2640\uFE0F"; // U+1F938 U+1F3FD U+200D U+2640 U+FE0F WOMAN CARTWHEELING: MEDIUM SKIN TONE

        // All valid scalars [ U+0000 .. U+D7FF ] and [ U+E000 .. U+10FFFF ].
        private static readonly IEnumerable<Rune> s_allValidScalars = Enumerable.Range(0x0000, 0xD800).Concat(Enumerable.Range(0xE000, 0x110000 - 0xE000)).Select(value => new Rune(value));

        private static readonly ReadOnlyMemory<uint> s_allScalarsAsUtf32;
        private static readonly ReadOnlyMemory<char> s_allScalarsAsUtf16;
        private static readonly ReadOnlyMemory<byte> s_allScalarsAsUtf8;
        // ReSharper restore InconsistentNaming UnusedMember.Local

        static CharacterEncodingTests()
        {
            var allScalarsAsUtf32 = new List<uint>();
            var allScalarsAsUtf16 = new List<char>();
            var allScalarsAsUtf8 = new List<byte>();

            Span<byte> utf8 = stackalloc byte[4];
            Span<char> utf16 = stackalloc char[2];

            foreach (Rune rune in s_allValidScalars)
            {
                int utf8Length = ToUtf8(rune, utf8);
                int utf16Length = ToUtf16(rune, utf16);

                allScalarsAsUtf32.Add((uint)rune.Value);

                for (int i = 0; i < utf16Length; i++)
                    allScalarsAsUtf16.Add(utf16[i]);

                for (int i = 0; i < utf8Length; i++)
                    allScalarsAsUtf8.Add(utf8[i]);
            }

            s_allScalarsAsUtf32 = allScalarsAsUtf32.ToArray().AsMemory();
            s_allScalarsAsUtf16 = allScalarsAsUtf16.ToArray().AsMemory();
            s_allScalarsAsUtf8 = allScalarsAsUtf8.ToArray().AsMemory();
        }

        /*
         * COMMON UTILITIES FOR UNIT TESTS
         */

        public static byte[] DecodeHex(ReadOnlySpan<char> inputHex)
        {
            Assert.True(Regex.IsMatch(inputHex.ToString(), "^([0-9a-fA-F]{2})*$"), "Input must be an even number of hex characters.");

            return Convert.FromHexString(inputHex);
        }

        public static byte[] ToUtf8(Rune rune)
        {
            Span<byte> utf8 = stackalloc byte[4];

            int length = ToUtf8(rune, utf8);
            return utf8.Slice(0, length).ToArray();
        }

        private static char[] ToUtf16(Rune rune)
        {
            Span<char> utf16 = stackalloc char[2];

            int length = ToUtf16(rune, utf16);
            return utf16.Slice(0, length).ToArray();
        }

        // !! IMPORTANT !!
        // Don't delete this implementation, as we use it as a reference to make sure the framework's
        // transcoding logic is correct.
        public static int ToUtf8(Rune rune, Span<byte> destination)
        {
            if (!Rune.IsValid(rune.Value))
            {
                Assert.True(Rune.IsValid(rune.Value), $"Rune with value U+{(uint)rune.Value:X4} is not well-formed.");
            }

            Assert.True(destination.Length == 4);

            destination[0] = 0;
            destination[1] = 0;
            destination[2] = 0;
            destination[3] = 0;

            if (rune.Value < 0x80)
            {
                destination[0] = (byte)rune.Value;
                return 1;
            }
            else if (rune.Value < 0x0800)
            {
                destination[0] = (byte)((rune.Value >> 6) | 0xC0);
                destination[1] = (byte)((rune.Value & 0x3F) | 0x80);
                return 2;
            }
            else if (rune.Value < 0x10000)
            {
                destination[0] = (byte)((rune.Value >> 12) | 0xE0);
                destination[1] = (byte)(((rune.Value >> 6) & 0x3F) | 0x80);
                destination[2] = (byte)((rune.Value & 0x3F) | 0x80);
                return 3;
            }
            else
            {
                destination[0] = (byte)((rune.Value >> 18) | 0xF0);
                destination[1] = (byte)(((rune.Value >> 12) & 0x3F) | 0x80);
                destination[2] = (byte)(((rune.Value >> 6) & 0x3F) | 0x80);
                destination[3] = (byte)((rune.Value & 0x3F) | 0x80);
                return 4;
            }
        }

        // !! IMPORTANT !!
        // Don't delete this implementation, as we use it as a reference to make sure the framework's
        // transcoding logic is correct.
        private static int ToUtf16(Rune rune, Span<char> destination)
        {
            if (!Rune.IsValid(rune.Value))
            {
                Assert.True(Rune.IsValid(rune.Value), $"Rune with value U+{(uint)rune.Value:X4} is not well-formed.");
            }

            Assert.True(destination.Length == 2);

            destination[0] = '\0';
            destination[1] = '\0';

            if (rune.IsBmp)
            {
                destination[0] = (char)rune.Value;
                return 1;
            }
            else
            {
                destination[0] = (char)((rune.Value >> 10) + 0xD800 - 0x40);
                destination[1] = (char)((rune.Value & 0x03FF) + 0xDC00);
                return 2;
            }
        }

        [Theory]
        [InlineData("", "")] // empty string is OK
        [InlineData(X_UTF16, X_UTF8)]
        [InlineData(E_ACUTE_UTF16, E_ACUTE_UTF8)]
        [InlineData(EURO_SYMBOL_UTF16, EURO_SYMBOL_UTF8)]
        public void Utf16ToUtf8_WithSmallValidBuffers(string utf16Input, string expectedUtf8TranscodingHex)
        {
            Assert.InRange(utf16Input.Length, 0, 1);

            Utf16ToUtf8_String_Test_Core(
                utf16Input: utf16Input,
                destinationSize: expectedUtf8TranscodingHex.Length / 2,
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf8Transcoding: DecodeHex(expectedUtf8TranscodingHex));
        }

        [Theory]
        [InlineData('\uD800', CharacterEncodingResult.InsufficientLength)] // standalone high surrogate
        [InlineData('\uDFFF', CharacterEncodingResult.InvalidFormat)] // standalone low surrogate
        public void Utf16ToUtf8_WithOnlyStandaloneSurrogates(char charValue, CharacterEncodingResult expectedEncodingResult)
        {
            Utf16ToUtf8_String_Test_Core(
                utf16Input: new[] { charValue },
                destinationSize: 0,
                expectedEncodingResult: expectedEncodingResult,
                expectedUtf8Transcoding: Span<byte>.Empty);
        }

        [Theory]
        [InlineData("<LOW><HIGH>", "")] // swapped surrogate pair characters
        [InlineData("A<LOW><HIGH>", "41")] // consume standalone ASCII char, then swapped surrogate pair characters
        [InlineData("A<HIGH>B", "41F0")] // consume standalone ASCII char, then standalone high surrogate char
        [InlineData("A<LOW>B", "41")] // consume standalone ASCII char, then standalone low surrogate char
        [InlineData("AB<HIGH><HIGH>", "4142F0")] // consume two ASCII chars, then standalone high surrogate char
        [InlineData("AB<LOW><LOW>", "4142")] // consume two ASCII chars, then standalone low surrogate char
        public void Utf16ToUtf8_WithInvalidSurrogates(string utf16Input, string expectedUtf8TranscodingHex)
        {
            // xUnit can't handle ill-formed strings in [InlineData], so we replace here.

            utf16Input = utf16Input.Replace("<HIGH>", "\uD800").Replace("<LOW>", "\uDFFF");

            // These test cases are for the "fast processing" code which is the main loop of TranscodeToUtf8,
            // so inputs should be at least 2 chars.

            Assert.True(utf16Input.Length >= 2);

            Utf16ToUtf8_String_Test_Core(
                utf16Input: utf16Input,
                destinationSize: expectedUtf8TranscodingHex.Length / 2,
                expectedEncodingResult: CharacterEncodingResult.InvalidFormat,
                expectedUtf8Transcoding: DecodeHex(expectedUtf8TranscodingHex));

            // Now try the tests again with a larger buffer.
            // This ensures that running out of destination space wasn't the reason we failed.

            Utf16ToUtf8_String_Test_Core(
                utf16Input: utf16Input,
                destinationSize: (expectedUtf8TranscodingHex.Length) / 2 + 16,
                expectedEncodingResult: CharacterEncodingResult.InvalidFormat,
                expectedUtf8Transcoding: DecodeHex(expectedUtf8TranscodingHex));
        }

        [Theory]
        [InlineData("80", CharacterEncodingResult.InsufficientLength, "")] // sequence cannot begin with continuation character
        [InlineData("8182", CharacterEncodingResult.InsufficientLength, "")] // sequence cannot begin with continuation character
        [InlineData("838485", CharacterEncodingResult.InsufficientLength, "")] // sequence cannot begin with continuation character
        [InlineData(X_UTF8 + "80", CharacterEncodingResult.InsufficientLength, X_UTF16)] // sequence cannot begin with continuation character
        [InlineData(X_UTF8 + "8182", CharacterEncodingResult.InsufficientLength, X_UTF16)] // sequence cannot begin with continuation character
        [InlineData("C0", CharacterEncodingResult.InvalidFormat, "")] // [ C0 ] is always invalid
        [InlineData("C080", CharacterEncodingResult.InsufficientLength, "")] // [ C0 ] is always invalid
        [InlineData("C08081", CharacterEncodingResult.InsufficientLength, "")] // [ C0 ] is always invalid
        [InlineData(X_UTF8 + "C1", CharacterEncodingResult.InvalidFormat, X_UTF16)] // [ C1 ] is always invalid
        [InlineData(X_UTF8 + "C180", CharacterEncodingResult.InsufficientLength, X_UTF16)] // [ C1 ] is always invalid
        [InlineData(X_UTF8 + "C27F", CharacterEncodingResult.InsufficientLength, X_UTF16)] // [ C2 ] is improperly terminated
        [InlineData("E2827F", CharacterEncodingResult.InsufficientLength, "")] // [ E2 82 ] is improperly terminated
        [InlineData("E09F80", CharacterEncodingResult.InsufficientLength, "")] // [ E0 9F ... ] is overlong
        [InlineData("E0C080", CharacterEncodingResult.InsufficientLength, "")] // [ E0 ] is improperly terminated
        [InlineData("ED7F80", CharacterEncodingResult.InsufficientLength, "")] // [ ED ] is improperly terminated
        [InlineData("EDA080", CharacterEncodingResult.InsufficientLength, "")] // [ ED A0 ... ] is surrogate
        public void Utf8ToUtf16_WithSmallInvalidBuffers(string utf8HexInput, CharacterEncodingResult expectedEncodingResult, string expectedUtf16Transcoding)
        {
            Utf8ToUtf16_String_Test_Core(
              utf8Input: DecodeHex(utf8HexInput),
              destinationSize: expectedUtf16Transcoding.Length,
              expectedEncodingResult: expectedEncodingResult,
              expectedUtf16Transcoding: expectedUtf16Transcoding);

            // Now try the tests again with a larger buffer.
            // This ensures that the sequence is seen as invalid when not hitting the destination length check.

            Utf8ToUtf16_String_Test_Core(
                utf8Input: DecodeHex(utf8HexInput),
                destinationSize: expectedUtf16Transcoding.Length + 16,
                expectedEncodingResult: CharacterEncodingResult.InvalidFormat,
                expectedUtf16Transcoding: expectedUtf16Transcoding);
        }

        [Theory]
        /* SMALL VALID BUFFERS - tests drain loop at end of method */
        [InlineData("")] // empty string is OK
        [InlineData("X")]
        [InlineData("XY")]
        [InlineData("XYZ")]
        [InlineData(E_ACUTE_UTF16)]
        [InlineData(X_UTF16 + E_ACUTE_UTF16)]
        [InlineData(E_ACUTE_UTF16 + X_UTF16)]
        [InlineData(EURO_SYMBOL_UTF16)]
        /* LARGE VALID BUFFERS - test main loop at beginning of method */
        [InlineData(E_ACUTE_UTF16 + "ABCD" + "0123456789:;<=>?")] // Loop unrolling at end of buffer
        [InlineData(E_ACUTE_UTF16 + "ABCD" + "0123456789:;<=>?" + "01234567" + E_ACUTE_UTF16 + "89:;<=>?")] // Loop unrolling interrupted by non-ASCII
        [InlineData("ABC" + E_ACUTE_UTF16 + "0123")] // 3 ASCII bytes followed by non-ASCII
        [InlineData("AB" + E_ACUTE_UTF16 + "0123")] // 2 ASCII bytes followed by non-ASCII
        [InlineData("A" + E_ACUTE_UTF16 + "0123")] // 1 ASCII byte followed by non-ASCII
        [InlineData(E_ACUTE_UTF16 + E_ACUTE_UTF16 + E_ACUTE_UTF16 + E_ACUTE_UTF16)] // 4x 2-byte sequences, exercises optimization code path in 2-byte sequence processing
        [InlineData(E_ACUTE_UTF16 + E_ACUTE_UTF16 + E_ACUTE_UTF16 + "PQ")] // 3x 2-byte sequences + 2 ASCII bytes, exercises optimization code path in 2-byte sequence processing
        [InlineData(E_ACUTE_UTF16 + "PQ")] // single 2-byte sequence + 2 trailing ASCII bytes, exercises draining logic in 2-byte sequence processing
        [InlineData(E_ACUTE_UTF16 + "P" + E_ACUTE_UTF16 + "0@P")] // single 2-byte sequences + 1 trailing ASCII byte + 2-byte sequence, exercises draining logic in 2-byte sequence processing
        [InlineData(EURO_SYMBOL_UTF16 + "@")] // single 3-byte sequence + 1 trailing ASCII byte, exercises draining logic in 3-byte sequence processing
        [InlineData(EURO_SYMBOL_UTF16 + "@P`")] // single 3-byte sequence + 3 trailing ASCII byte, exercises draining logic and "running out of data" logic in 3-byte sequence processing
        [InlineData(EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16)] // 3x 3-byte sequences, exercises "stay within 3-byte loop" logic in 3-byte sequence processing
        [InlineData(EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16)] // 4x 3-byte sequences, exercises "consume multiple bytes at a time" logic in 3-byte sequence processing
        [InlineData(EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16 + E_ACUTE_UTF16)] // 3x 3-byte sequences + single 2-byte sequence, exercises "consume multiple bytes at a time" logic in 3-byte sequence processing
        [InlineData(EURO_SYMBOL_UTF16 + EURO_SYMBOL_UTF16 + E_ACUTE_UTF16 + E_ACUTE_UTF16 + E_ACUTE_UTF16 + E_ACUTE_UTF16)] // 2x 3-byte sequences + 4x 2-byte sequences, exercises "consume multiple bytes at a time" logic in 3-byte sequence processing
        [InlineData(GRINNING_FACE_UTF16 + GRINNING_FACE_UTF16)] // 2x 4-byte sequences, exercises 4-byte sequence processing
        [InlineData(GRINNING_FACE_UTF16 + "@AB")] // single 4-byte sequence + 3 ASCII bytes, exercises 4-byte sequence processing and draining logic
        [InlineData(WOMAN_CARTWHEELING_MEDSKIN_UTF16)] // exercises switching between multiple sequence lengths
        public void Utf8ToUtf16_ValidBuffers(string utf16Input)
        {
            // We're going to run the tests with destination buffer lengths ranging from 0 all the way
            // to buffers large enough to hold the full output. This allows us to test logic that
            // detects whether we're about to overrun our destination buffer and instead returns DestinationTooSmall.

            Rune[] enumeratedScalars = utf16Input.EnumerateRunes().ToArray();

            // Convert entire input to UTF-8 using our unit test reference logic.

            byte[] utf8Input = enumeratedScalars.SelectMany(ToUtf8).ToArray();

            // 0-length buffer test
            Utf8ToUtf16_String_Test_Core(
                utf8Input: utf8Input,
                destinationSize: 0,
                expectedEncodingResult: (utf8Input.Length == 0) ? CharacterEncodingResult.Success : CharacterEncodingResult.InsufficientLength,
                expectedUtf16Transcoding: ReadOnlySpan<char>.Empty);

            char[] concatenatedUtf16 = Array.Empty<char>();

            for (int i = 0; i < enumeratedScalars.Length; i++)
            {
                Rune thisScalar = enumeratedScalars[i];

                // if this is an astral scalar value, quickly test a buffer that's not large enough to contain the entire UTF-16 encoding

                if (!thisScalar.IsBmp)
                {
                    Utf8ToUtf16_String_Test_Core(
                        utf8Input: utf8Input,
                        destinationSize: concatenatedUtf16.Length + 1,
                        expectedEncodingResult: CharacterEncodingResult.InsufficientLength,
                        expectedUtf16Transcoding: concatenatedUtf16);
                }

                // now provide a destination buffer large enough to hold the next full scalar encoding

                concatenatedUtf16 = concatenatedUtf16.Concat(ToUtf16(thisScalar)).ToArray();

                Utf8ToUtf16_String_Test_Core(
                    utf8Input: utf8Input,
                    destinationSize: concatenatedUtf16.Length,
                    expectedEncodingResult: (i == enumeratedScalars.Length - 1) ? CharacterEncodingResult.Success : CharacterEncodingResult.InsufficientLength,
                    expectedUtf16Transcoding: concatenatedUtf16);
            }

            // now throw lots of ASCII data at the beginning so that we exercise the vectorized code paths

            utf16Input = new string('x', 64) + utf16Input;
            utf8Input = utf16Input.EnumerateRunes().SelectMany(ToUtf8).ToArray();

            Utf8ToUtf16_String_Test_Core(
                utf8Input: utf8Input,
                destinationSize: utf16Input.Length,
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf16Transcoding: utf16Input);

            // now throw some non-ASCII data at the beginning so that we *don't* exercise the vectorized code paths

            utf16Input = WOMAN_CARTWHEELING_MEDSKIN_UTF16 + utf16Input[64..];
            utf8Input = utf16Input.EnumerateRunes().SelectMany(ToUtf8).ToArray();

            Utf8ToUtf16_String_Test_Core(
                utf8Input: utf8Input,
                destinationSize: utf16Input.Length,
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf16Transcoding: utf16Input);
        }

        [Fact]
        public void Utf8ToUtf16_String_AllPossibleScalarValues()
        {
            Utf8ToUtf16_String_Test_Core(
                utf8Input: s_allScalarsAsUtf8.Span,
                destinationSize: s_allScalarsAsUtf16.Length,
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf16Transcoding: s_allScalarsAsUtf16.Span);
        }

        [Fact]
        public void Utf16ToUtf8_String_AllPossibleScalarValues()
        {
            Utf16ToUtf8_String_Test_Core(
                utf16Input: s_allScalarsAsUtf16.Span,
                destinationSize: s_allScalarsAsUtf8.Length,
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf8Transcoding: s_allScalarsAsUtf8.Span);
        }

        [Fact]
        public void Utf8ToUtf32_String_AllPossibleScalarValues()
        {
            Utf8ToUtf32_String_Test_Core(
                utf8Input: s_allScalarsAsUtf8.Span,
                destinationSize: s_allScalarsAsUtf32.Length,
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf32Transcoding: s_allScalarsAsUtf32.Span);
        }

        [Fact]
        public void Utf32ToUtf8_String_AllPossibleScalarValues()
        {
            Utf32ToUtf8_String_Test_Core(
                utf32Input: s_allScalarsAsUtf32.Span,
                destinationSize: s_allScalarsAsUtf8.Length,
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf8Transcoding: s_allScalarsAsUtf8.Span);
        }

        [Fact]
        public void Utf8ToUtf16_Length_AllPossibleScalarValues()
        {
            Utf8ToUtf16_Length_Test_Core(
                // Skip the first code point because it's 0
                utf8Input: s_allScalarsAsUtf8.Span.Slice(1),
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf16Length: s_allScalarsAsUtf16.Length - 1);
        }

        [Fact]
        public void Utf16ToUtf8_Length_AllPossibleScalarValues()
        {
            Utf16ToUtf8_Length_Test_Core(
                utf16Input: s_allScalarsAsUtf16.Span.Slice(1),
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf8Length: s_allScalarsAsUtf8.Length - 1);
        }

        [Fact]
        public void Utf8ToUtf32_Length_AllPossibleScalarValues()
        {
            Utf8ToUtf32_Length_Test_Core(
                utf8Input: s_allScalarsAsUtf8.Span.Slice(1),
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf32Length: s_allScalarsAsUtf32.Length - 1);
        }

        [Fact]
        public void Utf32ToUtf8_Length_AllPossibleScalarValues()
        {
            Utf32ToUtf8_Length_Test_Core(
                utf32Input: s_allScalarsAsUtf32.Span.Slice(1),
                expectedEncodingResult: CharacterEncodingResult.Success,
                expectedUtf8Length: s_allScalarsAsUtf8.Length - 1);
        }

        [Fact]
        public void Utf8ToUtf16_Character_AllPossibleScalarValues()
        {
            Span<byte> utf8 = stackalloc byte[4];
            Span<char> utf16 = stackalloc char[2];

            foreach (Rune rune in s_allValidScalars)
            {
                ToUtf8(rune, utf8);
                ToUtf16(rune, utf16);

                Utf8ToUtf16_Character_Test_Core(
                    utf8Input: utf8,
                    expectedEncodingResult: CharacterEncodingResult.Success,
                    expectedUtf16Transcoding: utf16);
            }
        }

        [Fact]
        public void Utf16ToUtf8_Character_AllPossibleScalarValues()
        {
            Span<byte> utf8 = stackalloc byte[4];
            Span<char> utf16 = stackalloc char[2];

            foreach (Rune rune in s_allValidScalars)
            {
                ToUtf8(rune, utf8);
                ToUtf16(rune, utf16);

                Utf16ToUtf8_Character_Test_Core(
                    utf16Input: utf16,
                    expectedEncodingResult: CharacterEncodingResult.Success,
                    expectedUtf8Transcoding: utf8);
            }
        }

        [Fact]
        public void Utf8ToUtf32_Character_AllPossibleScalarValues()
        {
            Span<byte> utf8 = stackalloc byte[4];

            foreach (Rune rune in s_allValidScalars)
            {
                ToUtf8(rune, utf8);

                Utf8ToUtf32_Character_Test_Core(
                    utf8Input: utf8,
                    expectedEncodingResult: CharacterEncodingResult.Success,
                    expectedUtf32Transcoding: (uint)rune.Value);
            }
        }

        [Fact]
        public void Utf32ToUtf8_Character_AllPossibleScalarValues()
        {
            Span<byte> utf8 = stackalloc byte[4];

            foreach (Rune rune in s_allValidScalars)
            {
                ToUtf8(rune, utf8);

                Utf32ToUtf8_Character_Test_Core(
                    utf32Input: (uint)rune.Value,
                    expectedEncodingResult: CharacterEncodingResult.Success,
                    expectedUtf8Transcoding: utf8);
            }
        }

        [Fact]
        public void PickOutCharacterFromUtf8String_AllPossibleScalarValues()
        {
            byte[] expectedUtf8 = new byte[4];
            byte[] actualUtf8 = new byte[4];
            
            ReadOnlySpan<byte> utf8Values = s_allScalarsAsUtf8.Span.Slice(1);

            foreach (Rune rune in s_allValidScalars.Skip(1))
            {
                ToUtf8(rune, expectedUtf8);

                CharacterEncodingResult result = CharacterEncoding.PickOutCharacterFromUtf8String(actualUtf8, ref utf8Values);

                Assert.Equal(CharacterEncodingResult.Success, result);
                Assert.Equal((ReadOnlySpan<byte>)expectedUtf8, actualUtf8);
            }
        }

        private static void Utf8ToUtf16_String_Test_Core(ReadOnlySpan<byte> utf8Input, int destinationSize, CharacterEncodingResult expectedEncodingResult, ReadOnlySpan<char> expectedUtf16Transcoding)
        {
            char[] destination = new char[destinationSize];

            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertStringUtf8ToUtf16Native(destination, utf8Input, utf8Input.Length);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf16Transcoding, destination.AsSpan(0, expectedUtf16Transcoding.Length));
        }

        private static void Utf16ToUtf8_String_Test_Core(ReadOnlySpan<char> utf16Input, int destinationSize, CharacterEncodingResult expectedEncodingResult, ReadOnlySpan<byte> expectedUtf8Transcoding)
        {
            byte[] destination = new byte[destinationSize];

            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertStringUtf16NativeToUtf8(destination, utf16Input, utf16Input.Length);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf8Transcoding, destination.AsSpan(0, expectedUtf8Transcoding.Length));
        }

        private static void Utf8ToUtf32_String_Test_Core(ReadOnlySpan<byte> utf8Input, int destinationSize, CharacterEncodingResult expectedEncodingResult, ReadOnlySpan<uint> expectedUtf32Transcoding)
        {
            uint[] destination = new uint[destinationSize];

            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertStringUtf8ToUtf32(destination, utf8Input, utf8Input.Length);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf32Transcoding, destination.AsSpan(0, expectedUtf32Transcoding.Length));
        }

        private static void Utf32ToUtf8_String_Test_Core(ReadOnlySpan<uint> utf32Input, int destinationSize, CharacterEncodingResult expectedEncodingResult, ReadOnlySpan<byte> expectedUtf8Transcoding)
        {
            byte[] destination = new byte[destinationSize];

            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertStringUtf32ToUtf8(destination, utf32Input, utf32Input.Length);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf8Transcoding, destination.AsSpan(0, expectedUtf8Transcoding.Length));
        }

        private static void Utf8ToUtf16_Length_Test_Core(ReadOnlySpan<byte> utf8Input, CharacterEncodingResult expectedEncodingResult, int expectedUtf16Length)
        {
            CharacterEncodingResult actualEncodingResult = CharacterEncoding.GetLengthOfConvertedStringUtf8ToUtf16Native(out int actualLength, utf8Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf16Length, actualLength);
        }

        private static void Utf16ToUtf8_Length_Test_Core(ReadOnlySpan<char> utf16Input, CharacterEncodingResult expectedEncodingResult, int expectedUtf8Length)
        {
            CharacterEncodingResult actualEncodingResult = CharacterEncoding.GetLengthOfConvertedStringUtf16NativeToUtf8(out int actualLength, utf16Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf8Length, actualLength);
        }

        private static void Utf8ToUtf32_Length_Test_Core(ReadOnlySpan<byte> utf8Input, CharacterEncodingResult expectedEncodingResult, int expectedUtf32Length)
        {
            CharacterEncodingResult actualEncodingResult = CharacterEncoding.GetLengthOfConvertedStringUtf8ToUtf32(out int actualLength, utf8Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf32Length, actualLength);
        }

        private static void Utf32ToUtf8_Length_Test_Core(ReadOnlySpan<uint> utf32Input, CharacterEncodingResult expectedEncodingResult, int expectedUtf8Length)
        {
            CharacterEncodingResult actualEncodingResult = CharacterEncoding.GetLengthOfConvertedStringUtf32ToUtf8(out int actualLength, utf32Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf8Length, actualLength);
        }

        private static void Utf8ToUtf16_Character_Test_Core(ReadOnlySpan<byte> utf8Input, CharacterEncodingResult expectedEncodingResult, ReadOnlySpan<char> expectedUtf16Transcoding)
        {
            Span<char> destination = stackalloc char[2];

            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertCharacterUtf8ToUtf16Native(destination, utf8Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf16Transcoding, destination.Slice(0, expectedUtf16Transcoding.Length));

            for (int i = expectedUtf16Transcoding.Length; i < destination.Length; i++)
            {
                Assert.Equal(0, destination[i]);
            }
        }

        private static void Utf16ToUtf8_Character_Test_Core(ReadOnlySpan<char> utf16Input, CharacterEncodingResult expectedEncodingResult, ReadOnlySpan<byte> expectedUtf8Transcoding)
        {
            Span<byte> destination = stackalloc byte[4];

            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertCharacterUtf16NativeToUtf8(destination, utf16Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf8Transcoding, destination.Slice(0, expectedUtf8Transcoding.Length));

            for (int i = expectedUtf8Transcoding.Length; i < destination.Length; i++)
            {
                Assert.Equal(0, destination[i]);
            }
        }

        private static void Utf8ToUtf32_Character_Test_Core(ReadOnlySpan<byte> utf8Input, CharacterEncodingResult expectedEncodingResult, uint expectedUtf32Transcoding)
        {
            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertCharacterUtf8ToUtf32(out uint destination, utf8Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf32Transcoding, destination);
        }

        private static void Utf32ToUtf8_Character_Test_Core(uint utf32Input, CharacterEncodingResult expectedEncodingResult, ReadOnlySpan<byte> expectedUtf8Transcoding)
        {
            Span<byte> destination = stackalloc byte[4];

            CharacterEncodingResult actualEncodingResult = CharacterEncoding.ConvertCharacterUtf32ToUtf8(destination, utf32Input);

            Assert.Equal(expectedEncodingResult, actualEncodingResult);
            Assert.Equal(expectedUtf8Transcoding, destination.Slice(0, expectedUtf8Transcoding.Length));

            for (int i = expectedUtf8Transcoding.Length; i < destination.Length; i++)
            {
                Assert.Equal(0, destination[i]);
            }
        }
    }
}