diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7440193f..aefb7018 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,6 +10,9 @@ errorprone-core = "2.50.0" errorprone-gradle = "5.1.1" slf4j = "2.0.18" sqlite-jdbc = "3.49.1.0" +mockito-core = "5.23.0" +junit-jupiter = "6.1.3" +hamcrest = "3.0" [libraries] spigotapi = { module = "org.spigotmc:spigot-api", version.ref = "spigotapi" } @@ -20,6 +23,9 @@ folia-scheduler-wrapper = { module = "com.github.NahuLD.folia-scheduler-wrapper: errorprone-core = { module = "com.google.errorprone:error_prone_core", version.ref = "errorprone-core" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } +mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito-core" } +junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit-jupiter" } +hamcrest = { module = "org.hamcrest:hamcrest", version.ref = "hamcrest" } [plugins] paperweight = { id = "io.papermc.paperweight.userdev", version.ref = "paperweight" } diff --git a/plugin/build.gradle.kts b/plugin/build.gradle.kts index 371aa086..5e577f84 100644 --- a/plugin/build.gradle.kts +++ b/plugin/build.gradle.kts @@ -1,3 +1,6 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent + plugins { alias(libs.plugins.shadow) } @@ -7,6 +10,8 @@ repositories { maven("https://jitpack.io") } +val mockitoAgent: Configuration = configurations.create("mockitoAgent") + dependencies { compileOnly(libs.spigotapi) implementation(project(":openinvapi")) @@ -32,6 +37,10 @@ dependencies { implementation(libs.planarwrappers) implementation(libs.folia.scheduler.wrapper) compileOnly(libs.sqlite.jdbc) + + testImplementation(rootProject.libs.hamcrest) + testImplementation(libs.mockito.core) + mockitoAgent(libs.mockito.core) { isTransitive = false } } java { @@ -42,6 +51,27 @@ tasks.withType().configureEach { options.release = 21 } +tasks.withType().configureEach { + // Use as many cores as possible to run tests. + maxParallelForks = Runtime.getRuntime().availableProcessors() + // As Bukkit is very heavily statically initialized, don't reuse forks. + forkEvery = 1 + jvmArgs("-Xshare:off", "-javaagent:${mockitoAgent.asPath}") + testLogging { + showStackTraces = true + exceptionFormat = TestExceptionFormat.FULL + events(TestLogEvent.STANDARD_OUT) + } +} + +testing { + suites { + named("test") { + useJUnitJupiter(libs.junit.jupiter.get().version!!) + } + } +} + tasks.processResources { expand( mutableMapOf( diff --git a/plugin/src/main/java/com/lishid/openinv/util/FuzzyJaroWinkler.java b/plugin/src/main/java/com/lishid/openinv/util/FuzzyJaroWinkler.java new file mode 100644 index 00000000..60a5ebae --- /dev/null +++ b/plugin/src/main/java/com/lishid/openinv/util/FuzzyJaroWinkler.java @@ -0,0 +1,136 @@ +package com.lishid.openinv.util; + +import org.jetbrains.annotations.ApiStatus; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * A modified Jaro-Winkler similarity that includes a cost for ignoring case. + */ +@NullMarked +public class FuzzyJaroWinkler { + + // Bitmask for converting ASCII characters to upper case. + private static final int TO_UPPER_BITMASK = ~0x20; + + /** + * This is specifically not optimized for equal strings because it is intended for internal use in an area where + * the inputs are never equal. + * + * @param val1 the first string for comparison + * @param val2 the second string for comparison + * @return the result of the comparison, where higher is better + */ + @ApiStatus.Internal + public static double getSimilarity(@Nullable String val1, @Nullable String val2) { + // Inputs should never be null or empty, but who knows what people might do to their databases. + if (val1 == null || val1.isEmpty() || val2 == null || val2.isEmpty()) { + return 0.0; + } + + int[] chars1 = val1.codePoints().toArray(); + int[] chars2 = val2.codePoints().toArray(); + double jaro = getJaroSimilarity(chars1, chars2); + + // Winkler prefix + int prefix = 0; + for (int i = 0; i < 4; ++i) { + if (chars1[i] != chars2[i] && !isEqualNormalized(chars1[i], chars2[i])) { + break; + } + ++prefix; + } + + return jaro + (prefix * 0.1 * (1.0 - jaro)); + } + + private static double getJaroSimilarity(int[] chars1, int[] chars2) { + boolean[] matched1 = new boolean[chars1.length]; + boolean[] matched2 = new boolean[chars2.length]; + + double matches = getMatches(chars1, chars2, matched1, matched2); + + if (matches == 0) { + return 0.0; + } + + double fuzz = getFuzzCost(chars1, chars2, matched1, matched2); + + return (matches / chars1.length + matches / chars2.length + (matches - fuzz) / matches) / 3.0; + } + + private static int getMatches(int[] chars1, int[] chars2, boolean[] matched1, boolean[] matched2) { + int similarityWindow = Math.max(0, Math.max(chars1.length, chars2.length) / 2 - 1); + int matches = 0; + + for (int index1 = 0; index1 < chars1.length; index1++) { + int start = Math.max(0, index1 - similarityWindow); + int end = Math.min(chars2.length, index1 + similarityWindow + 1); + for (int index2 = start; index2 < end; index2++) { + if (matched2[index2]) { + continue; + } + if (chars1[index1] == chars2[index2] || isEqualNormalized(chars1[index1], chars2[index2])) { + matched1[index1] = true; + matched2[index2] = true; + ++matches; + break; + } + } + } + + return matches; + } + + private static boolean isEqualNormalized(int char1, int char2) { + // This is just fast ascii uppercase. Better normalization is expensive and would require that we compare twice, + // once against normalized and once against raw, because normalized does not necessarily match raw. + return (char1 & TO_UPPER_BITMASK) == (char2 & TO_UPPER_BITMASK) && char1 >= 'A' && char1 <= 'z'; + } + + /** + * This is a slightly-modified version of transposition cost in a traditional Jaro-Winkler implementation. + * Instead of counting all mismatches as 0.5, mismatches that normalize to the same character are counted as 0.05. + * + * @param chars1 the characters of the first string + * @param chars2 the characters of the second string + * @param matched1 the indices of matched characters in the first string + * @param matched2 the indices of matched characters in the second string + * @return the fuzz cost of 0.05 per normalization and 0.5 per transposition + */ + private static double getFuzzCost(int[] chars1, int[] chars2, boolean[] matched1, boolean[] matched2) { + double fuzzFactor = 0; + + int index2 = 0; + for (int index1 = 0; index1 < chars1.length; index1++) { + if (!matched1[index1]) { + continue; + } + // We know there are an equal number of matches in both strings. Since we found a match in the first, + // we can safely increment forward to the next match in the second without risking running out of bounds. + while (!matched2[index2]) { + ++index2; + } + + int char2 = chars2[index2]; + ++index2; + + // Full equals, move on. + if (chars1[index1] == char2) { + continue; + } + + // Fuzzy equals, add small cost. + if (isEqualNormalized(chars1[index1], char2)) { + fuzzFactor += 0.05; + continue; + } + + // Not equal, transposition occurred. + fuzzFactor += 0.5; + } + + return fuzzFactor; + } + +} diff --git a/plugin/src/main/java/com/lishid/openinv/util/PlayerLoader.java b/plugin/src/main/java/com/lishid/openinv/util/PlayerLoader.java index ba651b7b..199b59c3 100644 --- a/plugin/src/main/java/com/lishid/openinv/util/PlayerLoader.java +++ b/plugin/src/main/java/com/lishid/openinv/util/PlayerLoader.java @@ -221,8 +221,8 @@ private void updateMatches(@NotNull PlayerJoinEvent event) { Map.Entry entry = iterator.next(); String oldMatch = entry.getValue().name(); String lookup = entry.getKey(); - float oldMatchScore = StringMetric.compareJaroWinkler(lookup, oldMatch); - float newMatchScore = StringMetric.compareJaroWinkler(lookup, name); + double oldMatchScore = FuzzyJaroWinkler.getSimilarity(lookup, oldMatch); + double newMatchScore = FuzzyJaroWinkler.getSimilarity(lookup, name); // If new match exceeds old match, delete old match. if (newMatchScore > oldMatchScore) { diff --git a/plugin/src/main/java/com/lishid/openinv/util/StringMetric.java b/plugin/src/main/java/com/lishid/openinv/util/StringMetric.java deleted file mode 100644 index 8feaca81..00000000 --- a/plugin/src/main/java/com/lishid/openinv/util/StringMetric.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * This file is an amalgamation of code from the Simmetrics authors. - * The originals may be found here: - * https://github.com/Simmetrics/simmetrics/blob/master/simmetrics-core/src/main/java/org/simmetrics/metrics/JaroWinkler.java - * https://github.com/Simmetrics/simmetrics/blob/master/simmetrics-core/src/main/java/org/simmetrics/metrics/Jaro.java - * - * Copyright (C) 2014 - 2016 Simmetrics Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.lishid.openinv.util; - -public class StringMetric { - - public static float compareJaroWinkler(String a, String b) { - final float jaroScore = compareJaro(a, b); - - if (jaroScore < (float) 0.7) { - return jaroScore; - } - - String prefix = commonPrefix(a, b); - int prefixLength = Math.min(prefix.codePointCount(0, prefix.length()), 4); - - return jaroScore + (prefixLength * (float) 0.1 * (1.0f - jaroScore)); - - } - - private static float compareJaro(String a, String b) { - if (a.isEmpty() && b.isEmpty()) { - return 1.0f; - } - - if (a.isEmpty() || b.isEmpty()) { - return 0.0f; - } - - final int[] charsA = a.codePoints().toArray(); - final int[] charsB = b.codePoints().toArray(); - - // Intentional integer division to round down. - final int halfLength = Math.max(0, Math.max(charsA.length, charsB.length) / 2 - 1); - - final int[] commonA = getCommonCodePoints(charsA, charsB, halfLength); - final int[] commonB = getCommonCodePoints(charsB, charsA, halfLength); - - // commonA and commonB will always contain the same multi-set of - // characters. Because getCommonCharacters has been optimized, commonA - // and commonB are -1-padded. So in this loop we count transposition - // and use commonCharacters to determine the length of the multi-set. - float transpositions = 0; - int commonCharacters = 0; - for ( - int length = commonA.length; - commonCharacters < length && commonA[commonCharacters] > -1; - commonCharacters++ - ) { - if (commonA[commonCharacters] != commonB[commonCharacters]) { - transpositions++; - } - } - - if (commonCharacters == 0) { - return 0.0f; - } - - float aCommonRatio = commonCharacters / (float) charsA.length; - float bCommonRatio = commonCharacters / (float) charsB.length; - float transpositionRatio = (commonCharacters - transpositions / 2.0f) / commonCharacters; - - return (aCommonRatio + bCommonRatio + transpositionRatio) / 3.0f; - } - - /* - * Returns an array of code points from a within b. A character in b is - * counted as common when it is within separation distance from the position - * in a. - */ - private static int[] getCommonCodePoints(final int[] charsA, final int[] charsB, final int separation) { - final int[] common = new int[Math.min(charsA.length, charsB.length)]; - final boolean[] matched = new boolean[charsB.length]; - - // Iterate of string a and find all characters that occur in b within - // the separation distance. Mark any matches found to avoid - // duplicate matchings. - int commonIndex = 0; - for (int i = 0, length = charsA.length; i < length; i++) { - final int character = charsA[i]; - final int index = indexOf( - character, - charsB, - i - separation, - i + separation + 1, - matched - ); - if (index > -1) { - common[commonIndex++] = character; - matched[index] = true; - } - } - - if (commonIndex < common.length) { - common[commonIndex] = -1; - } - - // Both invocations will yield the same multi-set terminated by -1, so - // they can be compared for transposition without making a copy. - return common; - } - - /* - * Search for code point in buffer starting at fromIndex to toIndex - 1. - * - * Returns -1 when not found. - */ - private static int indexOf(int character, int[] buffer, int fromIndex, int toIndex, boolean[] matched) { - - // compare char with range of characters to either side - for (int j = Math.max(0, fromIndex), length = Math.min(toIndex, buffer.length); j < length; j++) { - // check if found - if (buffer[j] == character && !matched[j]) { - return j; - } - } - - return -1; - } - - private static String commonPrefix(CharSequence a, CharSequence b) { - int maxPrefixLength = Math.min(a.length(), b.length()); - - int p; - - p = 0; - while (p < maxPrefixLength && a.charAt(p) == b.charAt(p)) { - ++p; - } - - if (validSurrogatePairAt(a, p - 1) || validSurrogatePairAt(b, p - 1)) { - --p; - } - - return a.subSequence(0, p).toString(); - } - - private static boolean validSurrogatePairAt(CharSequence string, int index) { - return index >= 0 && index <= string.length() - 2 && Character.isHighSurrogate(string.charAt(index)) && Character.isLowSurrogate(string.charAt(index + 1)); - } - - private StringMetric() { - } - -} diff --git a/plugin/src/main/java/com/lishid/openinv/util/profile/OfflinePlayerProfileStore.java b/plugin/src/main/java/com/lishid/openinv/util/profile/OfflinePlayerProfileStore.java index 6fdb6284..0a756040 100644 --- a/plugin/src/main/java/com/lishid/openinv/util/profile/OfflinePlayerProfileStore.java +++ b/plugin/src/main/java/com/lishid/openinv/util/profile/OfflinePlayerProfileStore.java @@ -1,6 +1,6 @@ package com.lishid.openinv.util.profile; -import com.lishid.openinv.util.StringMetric; +import com.lishid.openinv.util.FuzzyJaroWinkler; import org.bukkit.Bukkit; import org.bukkit.OfflinePlayer; import org.jetbrains.annotations.NotNull; @@ -58,7 +58,7 @@ public void tryImport() { public @Nullable Profile getProfileInexact(@NotNull String search) { ProfileStore.warnMainThread(logger); - float bestMatch = 0.0F; + double bestMatch = 0.0F; Profile bestProfile = null; for (OfflinePlayer player : Bukkit.getOfflinePlayers()) { String name = player.getName(); @@ -67,7 +67,7 @@ public void tryImport() { return null; } - float currentMatch = StringMetric.compareJaroWinkler(name, name); + double currentMatch = FuzzyJaroWinkler.getSimilarity(search, name); if (currentMatch > bestMatch) { bestMatch = currentMatch; diff --git a/plugin/src/main/java/com/lishid/openinv/util/profile/sqlite/JaroWinklerFunction.java b/plugin/src/main/java/com/lishid/openinv/util/profile/sqlite/JaroWinklerFunction.java index 8974d48f..62cf6458 100644 --- a/plugin/src/main/java/com/lishid/openinv/util/profile/sqlite/JaroWinklerFunction.java +++ b/plugin/src/main/java/com/lishid/openinv/util/profile/sqlite/JaroWinklerFunction.java @@ -1,6 +1,6 @@ package com.lishid.openinv.util.profile.sqlite; -import com.lishid.openinv.util.StringMetric; +import com.lishid.openinv.util.FuzzyJaroWinkler; import org.sqlite.Function; import java.sql.SQLException; @@ -12,10 +12,8 @@ protected void xFunc() throws SQLException { if (args() != 2) { throw new SQLException("JaroWinkler(str, str) requires 2 arguments but got " + args()); } - String val1 = value_text(0); - String val2 = value_text(1); - result(StringMetric.compareJaroWinkler(val1 == null ? "" : val1, val2 == null ? "" : val2)); + result(FuzzyJaroWinkler.getSimilarity(value_text(0), value_text(1))); } } diff --git a/plugin/src/test/java/com/lishid/openinv/util/FuzzyJaroWinklerTest.java b/plugin/src/test/java/com/lishid/openinv/util/FuzzyJaroWinklerTest.java new file mode 100644 index 00000000..38154710 --- /dev/null +++ b/plugin/src/test/java/com/lishid/openinv/util/FuzzyJaroWinklerTest.java @@ -0,0 +1,41 @@ +package com.lishid.openinv.util; + +import org.jspecify.annotations.NullMarked; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@NullMarked +class FuzzyJaroWinklerTest { + + @ParameterizedTest + @MethodSource("getLookups") + void checkBestName(String lookup, String far, String near) { + double similarityFar = FuzzyJaroWinkler.getSimilarity(lookup, far); + double similarityNear = FuzzyJaroWinkler.getSimilarity(lookup, near); + + assertThat("Near must be better match than far", similarityNear, is(greaterThan(similarityFar))); + } + + private List getLookups() { + return Arrays.asList( + Arguments.of("johnminecraft", "frankminecraft", "johnminecraft123"), + Arguments.of("johnminecraft", "johnENIMCRAFT", "johnMINECRAFT"), + Arguments.of("johnminecraft", "johncavecraft", "JOHNMINECRAFT"), + Arguments.of("alice1234", "aleci1234", "ALICE1234"), + Arguments.of("ALICEMC", "ALICECM", "alicemc"), + Arguments.of("ALICEMINECRAFT", "aliceminceraft", "aliceminecraft"), + Arguments.of("aliceminecraf", "aliceminecraft", "ALICEMINECRAF") + ); + } + +}