Add regex support for filtered join addresses setting (#4342)

Author: Jsinco

Plan/build.gradle

@@ -90,7 +90,7 @@ subprojects {
         placeholderapiVersion = "2.11.5"
         nkPlaceholderapiVersion = "1.4-SNAPSHOT"
 
-        junitVersion = "5.13.4"
+        junitVersion = "6.0.1"
         mockitoVersion = "5.19.0"
         seleniumVersion = "4.35.0"
         testContainersVersion = "1.21.3"
@@ -121,7 +121,7 @@ subprojects {
         }
 
         // Required for JUnit to discover tests in modules.
-        testRuntimeOnly("org.junit.platform:junit-platform-launcher")
+        testRuntimeOnly("org.junit.platform:junit-platform-launcher:$junitVersion")
     }
 
     test {

Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/html/Contributors.java

@@ -118,7 +118,8 @@ public class Contributors {
             new Contributor("Zaemong", LANG),
             new Contributor("TWJohnJohn20116", LANG),
             new Contributor("YannicHock", CODE),
-            new Contributor("SaolGhra", CODE)
+            new Contributor("SaolGhra", CODE),
+            new Contributor("Jsinco", CODE)
     };
 
     private Contributors() {

Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/JSONFactory.java

@@ -52,12 +52,14 @@
 import com.djrapitops.plan.storage.database.sql.tables.JoinAddressTable;
 import com.djrapitops.plan.utilities.comparators.SessionStartComparator;
 import com.djrapitops.plan.utilities.dev.Untrusted;
+import com.djrapitops.plan.utilities.java.Lists;
 import com.djrapitops.plan.utilities.java.Maps;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
@@ -159,13 +161,13 @@ public List<RetentionData> networkPlayerRetentionAsJSONMap() {
         return db.query(PlayerRetentionQueries.fetchRetentionData());
     }
 
-    private static void removeFiltered(Map<UUID, String> addressByPlayerUUID, List<String> filteredJoinAddresses) {
-        if (filteredJoinAddresses.isEmpty() || filteredJoinAddresses.equals(List.of("play.example.com"))) return;
+    private static void removeFiltered(Map<UUID, String> addressByPlayerUUID, List<Pattern> filteredJoinAddresses) {
+        if (filteredJoinAddresses.isEmpty() || filteredJoinAddresses.equals(List.of(Pattern.compile("play\\.example\\.com")))) return;
 
         Set<UUID> toRemove = new HashSet<>();
         // Remove filtered addresses from the data
         for (Map.Entry<UUID, String> entry : addressByPlayerUUID.entrySet()) {
-            if (filteredJoinAddresses.contains(entry.getValue())) {
+            if (filteredJoinAddresses.stream().anyMatch(pattern -> pattern.matcher(entry.getValue()).matches())) {
                 toRemove.add(entry.getKey());
             }
         }
@@ -176,7 +178,7 @@ private static void removeFiltered(Map<UUID, String> addressByPlayerUUID, List<S
 
     public PlayerJoinAddresses playerJoinAddresses(ServerUUID serverUUID, boolean includeByPlayerMap) {
         Database db = dbSystem.getDatabase();
-        List<String> filteredJoinAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
+        List<Pattern> filteredJoinAddresses = Lists.map(config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES), Pattern::compile);
         if (includeByPlayerMap) {
             Map<UUID, String> addresses = db.query(JoinAddressQueries.latestJoinAddressesOfPlayers(serverUUID));
 
@@ -188,16 +190,20 @@ public PlayerJoinAddresses playerJoinAddresses(ServerUUID serverUUID, boolean in
             );
         } else {
             List<String> addresses = db.query(JoinAddressQueries.uniqueJoinAddresses(serverUUID));
-            addresses.removeAll(filteredJoinAddresses);
+            addresses.removeIf(address ->
+                    filteredJoinAddresses.stream().anyMatch(pattern -> pattern.matcher(address).matches())
+            );
             return new PlayerJoinAddresses(addresses, null);
         }
     }
 
     public PlayerJoinAddresses playerJoinAddresses(boolean includeByPlayerMap) {
         Database db = dbSystem.getDatabase();
-        List<String> filteredJoinAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
+        List<Pattern> filteredJoinAddresses = Lists.map(config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES), Pattern::compile);
         List<String> unique = db.query(JoinAddressQueries.uniqueJoinAddresses());
-        unique.removeAll(filteredJoinAddresses);
+        unique.removeIf(address ->
+                filteredJoinAddresses.stream().anyMatch(pattern -> pattern.matcher(address).matches())
+        );
         if (includeByPlayerMap) {
             Map<UUID, String> latest = db.query(JoinAddressQueries.latestJoinAddressesOfPlayers());
             removeFiltered(latest, filteredJoinAddresses);

Plan/common/src/main/java/com/djrapitops/plan/delivery/rendering/json/graphs/GraphJSONCreator.java

@@ -64,6 +64,7 @@
 import javax.inject.Singleton;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
@@ -474,11 +475,13 @@ public Map<String, Object> joinAddressesByDay(long after, long before, @Untruste
         return mapToJson(pieColors, joinAddresses);
     }
 
-    private static void removeFilteredAddresses(List<JoinAddressCount> addresses, List<String> filteredJoinAddresses) {
-        if (filteredJoinAddresses.isEmpty() || filteredJoinAddresses.equals(List.of("play.example.com"))) return;
+    private static void removeFilteredAddresses(List<JoinAddressCount> addresses, List<Pattern> filteredJoinAddresses) {
+        if (filteredJoinAddresses.isEmpty() || filteredJoinAddresses.equals(List.of(Pattern.compile("play\\.example\\.com")))) return;
 
         List<JoinAddressCount> addressesToRemove = addresses.stream()
-                .filter(address -> filteredJoinAddresses.contains(address.getJoinAddress()))
+                .filter(address ->
+                        filteredJoinAddresses.stream().anyMatch(p -> p.matcher(address.getJoinAddress()).matches())
+                )
                 .collect(Collectors.toList());
 
         if (!addressesToRemove.isEmpty()) {
@@ -505,7 +508,7 @@ private Map<String, Object> mapToJson(String[] pieColors, List<DateObj<Map<Strin
             translateUnknown(addressesByDate.getValue());
         }
 
-        List<String> filteredJoinAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
+        List<Pattern> filteredJoinAddresses = Lists.map(config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES), Pattern::compile);
 
         List<JoinAddressCounts> joinAddressCounts = joinAddresses.stream()
                 .map(addressesOnDay -> {

Plan/common/src/main/java/com/djrapitops/plan/gathering/JoinAddressValidator.java

@@ -19,14 +19,17 @@
 import com.djrapitops.plan.settings.config.PlanConfig;
 import com.djrapitops.plan.settings.config.paths.DataGatheringSettings;
 import com.djrapitops.plan.utilities.dev.Untrusted;
+import com.djrapitops.plan.utilities.java.Lists;
 import org.apache.commons.lang3.StringUtils;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.regex.Pattern;
 
 /**
  * Utility for validating and sanitizing join addresses.
@@ -37,7 +40,7 @@
 public class JoinAddressValidator {
 
     private final PlanConfig config;
-    private List<String> filteredAddresses;
+    private List<Pattern> filteredAddresses;
 
     @Inject
     public JoinAddressValidator(PlanConfig config) {
@@ -46,9 +49,13 @@ public JoinAddressValidator(PlanConfig config) {
     }
 
     private void prepareFilteredAddresses() {
-        if (filteredAddresses == null) {
-            filteredAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
-            if (filteredAddresses == null) filteredAddresses = new ArrayList<>();
+        if (filteredAddresses != null) return;
+
+        List<String> unCompiledFilteredAddresses = config.get(DataGatheringSettings.FILTER_JOIN_ADDRESSES);
+        if (unCompiledFilteredAddresses != null) {
+            filteredAddresses = Lists.map(unCompiledFilteredAddresses, Pattern::compile);
+        } else {
+            filteredAddresses = new ArrayList<>();
         }
     }
 
@@ -72,7 +79,8 @@ public String sanitize(@Untrusted String address) {
                 address = StringUtils.lowerCase(address);
             }
             prepareFilteredAddresses();
-            if (filteredAddresses.contains(address)) {
+            @Untrusted final String finalAddress = address;
+            if (filteredAddresses.stream().anyMatch(pattern -> pattern.matcher(finalAddress).matches())) {
                 address = "";
             }
         }

Plan/common/src/main/resources/assets/plan/bungeeconfig.yml

@@ -119,7 +119,7 @@ Data_gathering:
     # Does not affect already gathered data
     Preserve_case: false
     Preserve_invalid: false
-    # Replaces these join addresses with unknown
+    # Replaces these join addresses with unknown, supports regex
     Filter_out_from_data:
       - "play.example.com"
 # -----------------------------------------------------

Plan/common/src/main/resources/assets/plan/config.yml

@@ -123,7 +123,7 @@ Data_gathering:
     # Does not affect already gathered data
     Preserve_case: false
     Preserve_invalid: false
-    # Replaces these join addresses with unknown
+    # Replaces these join addresses with unknown, supports regex
     Filter_out_from_data:
       - "play.example.com"
 # -----------------------------------------------------