Add ColorCode.Core unit tests (#13)

ColorCode.Core/Compilation/Languages/Json.cs

@@ -13,7 +13,7 @@ namespace ColorCode.Compilation.Languages
     /// </summary>
     public class Json : ILanguage
     {
-        private const string Regex_String = @"""[^""\\]*(?:\\[^\r\n]|[^""\\]*)*""";
+        private const string Regex_String = @"""[^""\\]*(?:\\.[^""\\]*)*""";
         private const string Regex_Number = @"-?(?:0|[1-9][0-9]*)(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+)?";
 
         public string Id

ColorCode.Core/Compilation/Languages/Koka.cs

@@ -196,21 +196,21 @@ public IList<LanguageRule> Rules
                             }),
 
                     new LanguageRule(
-                        @"(?s)'(?:[^\t\n\\']+|(" + escape + @")|\\)*'",
+                        @"(?s)'[^\t\n\\']*(?:(?:(" + escape + @")|\\)[^\t\n\\']*)*'",
                         new Dictionary<int, string>
                             {
                                 { 0, ScopeName.String },
                                 { 1, ScopeName.StringEscape },
                             }),
                     new LanguageRule(
-                        @"(?s)@""(?:("""")|[^""]+)*""(?!"")",
+                        @"(?s)@""[^""]*(?:("""")[^""]*)*""(?!"")",
                         new Dictionary<int, string>
                             {
                                 { 0, ScopeName.StringCSharpVerbatim },
                                 { 1, ScopeName.StringEscape }
                             }),
                     new LanguageRule(
-                                @"(?s)""(?:[^\t\n\\""]+|(" + escape + @")|\\)*""",
+                                @"(?s)""[^\t\n\\""]*(?:(?:(" + escape + @")|\\)[^\t\n\\""]*)*""",
                                 new Dictionary<int, string>
                                     {
                                         { 0, ScopeName.String },

ColorCode.sln

@@ -32,6 +32,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ColorCode.UWP", "ColorCode.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColorCode.UWPTests", "Tests\ColorCode.UWPTests\ColorCode.UWPTests.csproj", "{A3D5A8A5-1D1F-412D-A829-C3DC1C7D4DF3}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColorCode.Core.UnitTests", "Tests\ColorCode.Core.UnitTests\ColorCode.Core.UnitTests.csproj", "{277FBC4B-50A4-440A-811F-39AEEB088978}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -201,6 +203,26 @@ Global
 		{A3D5A8A5-1D1F-412D-A829-C3DC1C7D4DF3}.Release|x86.ActiveCfg = Release|x86
 		{A3D5A8A5-1D1F-412D-A829-C3DC1C7D4DF3}.Release|x86.Build.0 = Release|x86
 		{A3D5A8A5-1D1F-412D-A829-C3DC1C7D4DF3}.Release|x86.Deploy.0 = Release|x86
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|ARM.ActiveCfg = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|ARM.Build.0 = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|ARM64.ActiveCfg = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|ARM64.Build.0 = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|x64.Build.0 = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Debug|x86.Build.0 = Debug|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|Any CPU.Build.0 = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|ARM.ActiveCfg = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|ARM.Build.0 = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|ARM64.ActiveCfg = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|ARM64.Build.0 = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|x64.ActiveCfg = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|x64.Build.0 = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|x86.ActiveCfg = Release|Any CPU
+		{277FBC4B-50A4-440A-811F-39AEEB088978}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -210,6 +232,7 @@ Global
 		{F1ED6BD6-2690-4BF3-835C-C3927C33424F} = {5B9F207C-2EAB-4F77-95C7-206D65C87137}
 		{DD20D31A-915E-43A2-B819-3A7AE39CA25C} = {5B9F207C-2EAB-4F77-95C7-206D65C87137}
 		{A3D5A8A5-1D1F-412D-A829-C3DC1C7D4DF3} = {5B9F207C-2EAB-4F77-95C7-206D65C87137}
+		{277FBC4B-50A4-440A-811F-39AEEB088978} = {5B9F207C-2EAB-4F77-95C7-206D65C87137}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {5A764590-3191-47F4-9257-5D5F63BC5713}

Tests/ColorCode.Core.UnitTests/ColorCode.Core.UnitTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <Nullable>disable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\ColorCode.HTML\ColorCode.HTML.csproj" />
+  </ItemGroup>
+
+</Project>

Tests/ColorCode.Core.UnitTests/HighlightRobustnessTests.cs

@@ -0,0 +1,60 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace ColorCode.Core.UnitTests
+{
+    /// <summary>
+    /// Robustness guard: highlighting malformed / partial / oversized input for any language must
+    /// complete quickly and never hang. Generalizes the JSON regression (issue #45) across the whole
+    /// language set, so a future catastrophic-backtracking rule is caught by the suite (issue #13).
+    /// </summary>
+    public class HighlightRobustnessTests
+    {
+        // Structurally awkward inputs that a real document can contain: empty / whitespace, unbalanced
+        // delimiters, oversized tokens, unterminated comments, and the partial-JSON array that used to
+        // send Json.cs into exponential-time matching (#45). None of these should take measurable time.
+        public static readonly string[] MalformedInputs =
+        {
+            "",
+            "   \n\t\r  ",
+            new string('{', 500),
+            new string('a', 5000),
+            "// " + new string('x', 3000),
+            "/* " + new string('y', 3000),
+            "<" + new string('a', 2000),
+            "[\n  { \"field\": \"" + new string('a', 60) + "\n",
+            string.Join(" ", Enumerable.Repeat("a=b,c;", 500)),
+        };
+
+        public static IEnumerable<object[]> LanguageAndInput =>
+            from language in Languages.All
+            from index in Enumerable.Range(0, MalformedInputs.Length)
+            select new object[] { language.Id, index };
+
+        [Theory]
+        [MemberData(nameof(LanguageAndInput))]
+        public void Highlighting_malformed_input_completes_quickly(string id, int inputIndex)
+        {
+            var language = Languages.FindById(id);
+            var input = MalformedInputs[inputIndex];
+
+            // Bound the call: a pre-fix catastrophic rule would never return, so the test failing fast
+            // (rather than hanging the whole run) IS the signal. A healthy rule finishes in milliseconds.
+            string html = null;
+            var task = Task.Run(() => html = new HtmlClassFormatter().GetHtmlString(input, language));
+
+            Assert.True(
+                task.Wait(TimeSpan.FromSeconds(5)),
+                $"Highlighting malformed input did not finish in 5s (language '{id}', input #{inputIndex}, " +
+                $"length {input.Length}) — likely catastrophic regex backtracking.");
+            Assert.NotNull(html);
+        }
+    }
+}

Tests/ColorCode.Core.UnitTests/HtmlClassFormatterTests.cs

@@ -0,0 +1,54 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Xunit;
+
+namespace ColorCode.Core.UnitTests
+{
+    /// <summary>
+    /// Output tests for <see cref="HtmlClassFormatter"/>: tokens are wrapped in spans carrying the
+    /// expected CSS class, the container div carries the language class, source text is HTML-encoded,
+    /// and a stylesheet is produced (issue #13). Assertions are structural, not exact-markup, so they
+    /// do not couple the suite to incidental formatting.
+    /// </summary>
+    public class HtmlClassFormatterTests
+    {
+        [Fact]
+        public void Json_keys_and_strings_get_their_scope_classes()
+        {
+            var html = new HtmlClassFormatter().GetHtmlString("{ \"name\": \"John\" }", Languages.FindById("json"));
+
+            Assert.Contains("class=\"json\"", html);   // container div uses the language's CssClassName
+            Assert.Contains("jsonKey", html);
+            Assert.Contains("jsonString", html);
+            Assert.Contains("John", html);
+        }
+
+        [Fact]
+        public void CSharp_keywords_get_the_keyword_class()
+        {
+            var html = new HtmlClassFormatter().GetHtmlString("public class Foo { }", Languages.CSharp);
+
+            Assert.Contains("class=\"keyword\"", html);
+        }
+
+        [Fact]
+        public void Source_text_is_html_encoded()
+        {
+            var html = new HtmlClassFormatter().GetHtmlString("if (a < b && c) { }", Languages.CSharp);
+
+            Assert.Contains("&lt;", html);
+            Assert.Contains("&amp;", html);
+        }
+
+        [Fact]
+        public void Css_stylesheet_is_produced()
+        {
+            var css = new HtmlClassFormatter().GetCSSString();
+
+            Assert.False(string.IsNullOrWhiteSpace(css));
+            Assert.Contains("{", css);
+        }
+    }
+}

Tests/ColorCode.Core.UnitTests/JsonBacktrackingTests.cs

@@ -0,0 +1,49 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace ColorCode.Core.UnitTests
+{
+    /// <summary>
+    /// Regression tests for the JSON language's <c>Regex_String</c> catastrophic backtracking
+    /// (issue #45): malformed JSON must not send the highlighter into exponential-time matching.
+    /// </summary>
+    public class JsonBacktrackingTests
+    {
+        [Fact]
+        public void Partial_json_does_not_catastrophically_backtrack()
+        {
+            // A JSON array whose last string value is unterminated. With the old Regex_String the
+            // key rule explores O(2^n) ways to partition the string body and never returns; with
+            // the fix it is linear. Bound the call so the pre-fix state fails fast (this test
+            // running long IS the bug) instead of hanging the whole run.
+            var partial = "[\n  { \"field\": \"" + new string('a', 40) + "\n";
+
+            string html = null;
+            var task = Task.Run(() => html = new HtmlClassFormatter().GetHtmlString(partial, Languages.FindById("json")));
+
+            Assert.True(
+                task.Wait(TimeSpan.FromSeconds(10)),
+                "Highlighting partial JSON did not finish in 10s — Regex_String is backtracking catastrophically (issue #45).");
+            Assert.NotNull(html);
+        }
+
+        [Fact]
+        public void Valid_json_still_highlights_keys_strings_and_escapes()
+        {
+            // The fix must not change how valid JSON is tokenised: keys, string values, and strings
+            // containing escaped quotes/backslashes all still colour correctly.
+            var json = "{ \"name\": \"John Doe\", \"path\": \"C:\\\\a\\\"b\" }";
+
+            var html = new HtmlClassFormatter().GetHtmlString(json, Languages.FindById("json"));
+
+            Assert.Contains("jsonKey", html);
+            Assert.Contains("jsonString", html);
+            Assert.Contains("John Doe", html);
+        }
+    }
+}

Tests/ColorCode.Core.UnitTests/KokaBacktrackingTests.cs

@@ -0,0 +1,51 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace ColorCode.Core.UnitTests
+{
+    /// <summary>
+    /// Regression tests for catastrophic backtracking in the Koka language's string rules. The
+    /// normal string, verbatim string, and char-literal patterns each used a nested quantifier
+    /// (<c>(?:X+|...)*</c>) that went exponential on an unterminated literal; the unrolled-loop
+    /// form fixes it. Same class of bug as the JSON one (issue #45).
+    /// </summary>
+    public class KokaBacktrackingTests
+    {
+        private static ILanguage Koka => Languages.FindById("koka");
+
+        [Theory]
+        [InlineData("\"")]  // unterminated normal string
+        [InlineData("@\"")] // unterminated verbatim string
+        [InlineData("'")]   // unterminated char literal
+        public void Unterminated_literal_does_not_catastrophically_backtrack(string opener)
+        {
+            // An unterminated literal followed by a long run of ordinary characters. With the old
+            // nested-quantifier rules this never returns; with the unrolled form it is linear. Bound
+            // the call so the pre-fix state fails fast instead of hanging the whole run.
+            var input = opener + new string('a', 200) + "\n";
+
+            string html = null;
+            var task = Task.Run(() => html = new HtmlClassFormatter().GetHtmlString(input, Koka));
+
+            Assert.True(
+                task.Wait(TimeSpan.FromSeconds(5)),
+                $"Highlighting an unterminated Koka literal (opener '{opener}') did not finish in 5s — catastrophic backtracking.");
+            Assert.NotNull(html);
+        }
+
+        [Fact]
+        public void Valid_string_still_highlights_with_the_string_scope()
+        {
+            // The fix must not change how a valid string tokenises.
+            var html = new HtmlClassFormatter().GetHtmlString("\"hello world\"", Koka);
+
+            Assert.Contains("class=\"string\"", html);
+            Assert.Contains("hello world", html);
+        }
+    }
+}

Tests/ColorCode.Core.UnitTests/LanguagesSmokeTests.cs

@@ -0,0 +1,60 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Xunit;
+
+namespace ColorCode.Core.UnitTests
+{
+    /// <summary>
+    /// Smoke tests over the built-in language registry: every language is retrievable and
+    /// all of its rule patterns are valid, compilable regular expressions. A malformed pattern
+    /// then fails here at test time rather than at a consumer's runtime (issue #13).
+    /// </summary>
+    public class LanguagesSmokeTests
+    {
+        public static IEnumerable<object[]> AllLanguageIds =>
+            Languages.All.Select(language => new object[] { language.Id });
+
+        [Fact]
+        public void Registry_exposes_languages()
+        {
+            Assert.NotEmpty(Languages.All);
+        }
+
+        [Theory]
+        [MemberData(nameof(AllLanguageIds))]
+        public void Language_has_identity_and_is_findable_by_id(string id)
+        {
+            var language = Languages.FindById(id);
+
+            Assert.NotNull(language);
+            Assert.False(string.IsNullOrWhiteSpace(language.Id));
+            Assert.False(string.IsNullOrWhiteSpace(language.Name));
+            Assert.Equal(id, language.Id);
+        }
+
+        [Theory]
+        [MemberData(nameof(AllLanguageIds))]
+        public void Language_rule_patterns_compile(string id)
+        {
+            var language = Languages.FindById(id);
+
+            Assert.NotNull(language.Rules);
+            Assert.NotEmpty(language.Rules);
+
+            foreach (var rule in language.Rules)
+            {
+                // Constructing the Regex validates the pattern; an invalid pattern throws and
+                // fails the test, naming the offending language id and pattern.
+                var exception = Record.Exception(() => new Regex(rule.Regex));
+                Assert.True(
+                    exception == null,
+                    $"Language '{id}' has an invalid rule pattern: {rule.Regex}\n{exception}");
+            }
+        }
+    }
+}