diff --git a/.github/workflows/java-17-builds.yml b/.github/workflows/java-17-builds.yml deleted file mode 100644 index 7aded923f67..00000000000 --- a/.github/workflows/java-17-builds.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Java 17 CI (MC 1.20.4) - -on: - push: - branches: - - master - - 'dev/**' - pull_request: - -jobs: - build: - if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - with: - submodules: recursive - - name: validate gradle wrapper - uses: gradle/actions/wrapper-validation@v4 - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - cache: gradle - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - name: Build Skript and run test scripts - run: ./gradlew clean skriptTestJava17 - - name: Upload Nightly Build - uses: actions/upload-artifact@v4 - if: success() - with: - name: skript-nightly - path: build/libs/* diff --git a/.github/workflows/java-21-builds.yml b/.github/workflows/java-21-builds.yml index 24fecb17aff..55b067a123b 100644 --- a/.github/workflows/java-21-builds.yml +++ b/.github/workflows/java-21-builds.yml @@ -1,4 +1,4 @@ -name: Java 21 CI (MC 1.20.6+) +name: Java 21 CI on: push: @@ -8,9 +8,20 @@ on: pull_request: jobs: - build: + prepare: + name: Parallelize Tests if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')" + uses: ./.github/workflows/parallelize-tests.yml + with: + environments: 1.21.3,1.21.4,1.21.5,1.21.8,1.21.10 + java_version: 21 + + build: + name: ${{ matrix.display }} + needs: prepare runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - uses: actions/checkout@v5 with: @@ -26,10 +37,10 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build Skript and run test scripts - run: ./gradlew clean skriptTestJava21 + run: ./gradlew clean customTest -PtestEnvs="${{ matrix.envs }}" -PtestEnvJavaVersion=21 - name: Upload Nightly Build uses: actions/upload-artifact@v4 - if: success() + if: success() && matrix.id == 1 with: name: skript-nightly path: build/libs/* diff --git a/.github/workflows/junit-17-builds.yml b/.github/workflows/junit-17-builds.yml deleted file mode 100644 index 1b8187f1087..00000000000 --- a/.github/workflows/junit-17-builds.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: JUnit (MC 1.20.4) - -on: - push: - branches: - - master - - 'dev/**' - pull_request: - -jobs: - build: - if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - with: - submodules: recursive - - name: validate gradle wrapper - uses: gradle/actions/wrapper-validation@v4 - - name: Set up JDK 21 - uses: actions/setup-java@v5 - with: - java-version: '21' - distribution: 'adopt' - cache: gradle - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - name: Build Skript and run JUnit - run: ./gradlew clean JUnitJava17 diff --git a/.github/workflows/junit-21-builds.yml b/.github/workflows/junit-21-builds.yml index 52fb32ee22f..675c40d17a1 100644 --- a/.github/workflows/junit-21-builds.yml +++ b/.github/workflows/junit-21-builds.yml @@ -1,4 +1,4 @@ -name: JUnit (MC 1.20.6+) +name: JUnit 21 on: push: @@ -8,9 +8,20 @@ on: pull_request: jobs: - build: + prepare: + name: Parallelize Tests if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')" + uses: ./.github/workflows/parallelize-tests.yml + with: + environments: 1.21.3,1.21.4,1.21.5,1.21.8,1.21.10 + java_version: 21 + + build: + name: ${{ matrix.display }} + needs: prepare runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - uses: actions/checkout@v5 with: @@ -25,5 +36,12 @@ jobs: cache: gradle - name: Grant execute permission for gradlew run: chmod +x gradlew - - name: Build Skript and run JUnit - run: ./gradlew clean JUnitJava21 + - name: Build Skript and run test scripts + run: ./gradlew clean customTest -PtestEnvs="${{ matrix.envs }}" -PtestEnvJavaVersion=21 -Pjunit=true + - name: Upload Nightly Build + uses: actions/upload-artifact@v4 + if: success() && matrix.id == 1 + with: + name: skript-nightly + path: build/libs/* + diff --git a/.github/workflows/parallelize-tests.yml b/.github/workflows/parallelize-tests.yml new file mode 100644 index 00000000000..c751627c6a0 --- /dev/null +++ b/.github/workflows/parallelize-tests.yml @@ -0,0 +1,85 @@ +name: Parallelize Tests + +on: + workflow_call: + inputs: + environments: + required: true + type: string + parallel_jobs: + required: false + type: number + default: 2 + java_version: + required: true + type: number + outputs: + matrix: + description: "Generated test matrix" + value: ${{ jobs.prepare.outputs.matrix }} + +jobs: + prepare: + name: "" + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Generate matrix + id: set-matrix + run: | + # Use environment variables + N=${{ inputs.parallel_jobs }} + JAVA_VERSION=${{ inputs.java_version }} + + # Convert to array and transform + IFS=',' read -ra ENVS <<< "${{ inputs.environments }}" + TRANSFORMED=() + for env in "${ENVS[@]}"; do + TRANSFORMED+=("java${JAVA_VERSION}/paper-${env}") + done + + TOTAL=${#TRANSFORMED[@]} + + # avoid creating more jobs than needed + JOBS=$((TOTAL < N ? TOTAL : N)) + # environments per job + BASE=$((TOTAL / JOBS)) + EXTRA=$((TOTAL % JOBS)) + + # Build matrix JSON + # Format is: + # [{"id":1,"envs":"java21/paper-1.20.6,java21/paper-1.21.3","display":"1.20.6, 1.21.3"},...] + # Where "envs" is the actual environment strings and "display" is for easier identification + MATRIX="[" + INDEX=0 + for ((i=0; i> $GITHUB_OUTPUT + echo "Generated matrix: {\"include\":$MATRIX}" diff --git a/build.gradle b/build.gradle index adff0ed54e4..56070bc9968 100644 --- a/build.gradle +++ b/build.gradle @@ -176,7 +176,7 @@ tasks.register('testNaming') { } enum Modifiers { - DEV_MODE, GEN_NIGHTLY_DOCS, GEN_RELEASE_DOCS, DEBUG, PROFILE, JUNIT + DEV_MODE, GEN_NIGHTLY_DOCS, GEN_RELEASE_DOCS, DEBUG, PROFILE, JUNIT, PARALLEL } // Create a test task with given name, environments dir/file, dev mode and java version. @@ -187,6 +187,7 @@ void createTestTask(String name, String desc, String environments, int javaVersi boolean junit = modifiers.contains(Modifiers.JUNIT) boolean releaseDocs = modifiers.contains(Modifiers.GEN_RELEASE_DOCS) boolean docs = modifiers.contains(Modifiers.GEN_NIGHTLY_DOCS) || releaseDocs + boolean parallel = modifiers.contains(Modifiers.PARALLEL) def artifact = 'build' + File.separator + 'libs' + File.separator if (junit) { artifact += 'Skript-JUnit.jar' @@ -227,7 +228,8 @@ void createTestTask(String name, String desc, String environments, int javaVersi junit, modifiers.contains(Modifiers.DEBUG), project.findProperty('verbosity') ?: "null", - timeout + timeout, + parallel ] // Do first is used when throwing exceptions. @@ -265,7 +267,7 @@ compileJava.options.encoding = 'UTF-8' compileTestJava.options.encoding = 'UTF-8' // Register different Skript testing tasks -String environments = 'src/test/skript/environments/'; +String environments = 'src/test/skript/environments/' int envJava = project.property('testEnvJavaVersion') == null ? latestJava : Integer.parseInt(project.property('testEnvJavaVersion') as String) createTestTask('quickTest', 'Runs tests on one environment being the latest supported Java and Minecraft.', environments + env, latestJava, 0) createTestTask('skriptTestJava21', 'Runs tests on all Java 21 environments.', environments + 'java21', java21, 0) @@ -287,6 +289,34 @@ tasks.register('JUnit') { dependsOn JUnitJava17, JUnitJava21 } +// custom test task +// usage: gradle customTest -PtestEnvs= -PtestEnvJavaVersion= -Ptimeout= -Pjunit=[true|false] -Pparallel=[true|false] +// defaults: testEnvJavaVersion=latestJava, timeout=0, junit=false, parallel=false +// example: gradle customTest -PtestEnvs=java21/1.21.4,java21/1.21.6 -PtestEnvJavaVersion=21 -Ptimeout=600000 -Pjunit=true -Pparallel=true + +// get environments +String propEnvs = project.hasProperty('testEnvs') ? project.property('testEnvs') as String : project.testEnv +String[] envList = propEnvs != null ? propEnvs.split(',') : [env] +StringBuilder customEnvironmentsBuilder = new StringBuilder() +for (int i = 0; i < envList.length; i++) { + customEnvironmentsBuilder.append(environments).append(envList[i]).append('.json') + if (i < envList.length - 1) { + customEnvironmentsBuilder.append(',') // use ',' as separator for multiple environments + } +} +String customEnvironments = customEnvironmentsBuilder.toString() + +long customTimeout = project.hasProperty('timeout') ? Long.parseLong(project.property('timeout') as String) : 0 +List customModifiers = new ArrayList<>() +if (project.hasProperty('junit') && (project.property('junit') as String).toLowerCase() == 'true') { + customModifiers.add(Modifiers.JUNIT) +} +if (project.hasProperty('parallel') && (project.property('parallel') as String).toLowerCase() == 'true') { + customModifiers.add(Modifiers.PARALLEL) +} +createTestTask('customTest', 'Runs tests based on provided parameters.', customEnvironments, envJava, customTimeout, customModifiers.toArray(new Modifiers[0]) as Modifiers[]) + + // Build flavor configurations task githubResources(type: ProcessResources) { from 'src/main/resources', { diff --git a/src/main/java/ch/njol/skript/test/platform/Environment.java b/src/main/java/ch/njol/skript/test/platform/Environment.java index 7588bbd2122..2331776e7c1 100644 --- a/src/main/java/ch/njol/skript/test/platform/Environment.java +++ b/src/main/java/ch/njol/skript/test/platform/Environment.java @@ -5,10 +5,7 @@ import com.google.gson.JsonObject; import org.jetbrains.annotations.Nullable; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.*; import java.lang.ProcessBuilder.Redirect; import java.net.URI; import java.net.URISyntaxException; @@ -21,12 +18,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; +import java.util.*; /** * Test environment information. @@ -158,7 +150,7 @@ public String getName() { return name; } - public void initialize(Path dataRoot, Path runnerRoot, boolean remake) throws IOException { + public void initialize(Path dataRoot, Path runnerRoot, boolean remake, int parallelPort) throws IOException { Path env = runnerRoot.resolve(name); boolean onlyCopySkript = Files.exists(env) && !remake; @@ -181,6 +173,21 @@ public void initialize(Path dataRoot, Path runnerRoot, boolean remake) throws IO Path source = dataRoot.resolve(resource.getSource()); Path target = env.resolve(resource.getTarget()); Files.createDirectories(target.getParent()); + // Copy file, but if the resource is server.properties, modify it to use a different port for parallel tests + if (resource.getTarget().endsWith("server.properties") && parallelPort > 0) { + List lines = Files.readAllLines(source, StandardCharsets.UTF_8); + List modifiedLines = new ArrayList<>(); + for (String line : lines) { + if (line.startsWith("server-port=")) { + modifiedLines.add("server-port=" + parallelPort); + } else { + modifiedLines.add(line); + } + } + Files.write(target, modifiedLines, StandardCharsets.UTF_8); + continue; + } + // normal copy Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); } @@ -232,8 +239,8 @@ public TestResults runTests(Path runnerRoot, Path testsRoot, boolean devMode, bo Process process = new ProcessBuilder(args) .directory(env.toFile()) - .redirectOutput(Redirect.INHERIT) - .redirectError(Redirect.INHERIT) +// .redirectOutput(Redirect.INHERIT) +// .redirectError(Redirect.INHERIT) .redirectInput(Redirect.INHERIT) .start(); @@ -253,7 +260,37 @@ public void run() { }, timeout); } + // Create threads to read and prefix the output + Thread outThread = new Thread(() -> { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + System.out.println("[" + this.getName() + "] " + line); + } + } catch (IOException e) { + e.printStackTrace(); + } + }); + + Thread errThread = new Thread(() -> { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getErrorStream()))) { + String line; + while ((line = reader.readLine()) != null) { + System.err.println("[" + this.getName() + "] " + line); + } + } catch (IOException e) { + e.printStackTrace(); + } + }); + + outThread.start(); + errThread.start(); + int code = process.waitFor(); + outThread.join(); + errThread.join(); if (code != 0) throw new IOException("environment returned with code " + code); diff --git a/src/main/java/ch/njol/skript/test/platform/PlatformMain.java b/src/main/java/ch/njol/skript/test/platform/PlatformMain.java index 744caa675b3..d52b20fb3e7 100644 --- a/src/main/java/ch/njol/skript/test/platform/PlatformMain.java +++ b/src/main/java/ch/njol/skript/test/platform/PlatformMain.java @@ -1,7 +1,6 @@ package ch.njol.skript.test.platform; import ch.njol.skript.test.utils.TestResults; -import ch.njol.util.NonNullPair; import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -9,20 +8,11 @@ import org.apache.commons.lang.StringUtils; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; /** @@ -36,13 +26,16 @@ public static void main(String... args) throws IOException, InterruptedException Gson gson = new GsonBuilder().setPrettyPrinting().create(); Path runnerRoot = Paths.get(args[0]); - assert runnerRoot != null; Path testsRoot = Paths.get(args[1]).toAbsolutePath(); - assert testsRoot != null; Path dataRoot = Paths.get(args[2]); - assert dataRoot != null; - Path envsRoot = Paths.get(args[3]); - assert envsRoot != null; + // allow multiple environments separated by commas + List envPaths = new ArrayList<>(); + String envsArg = args[3]; + envsArg = envsArg.trim(); + String[] envPathStrings = envsArg.split(","); + for (String envPath : envPathStrings) { + envPaths.add(Paths.get(envPath.trim())); + } boolean devMode = "true".equals(args[4]); boolean genDocs = "true".equals(args[5]); boolean jUnit = "true".equals(args[6]); @@ -51,58 +44,93 @@ public static void main(String... args) throws IOException, InterruptedException long timeout = Long.parseLong(args[9]); if (timeout < 0) timeout = 0; - Set jvmArgs = Sets.newHashSet(Arrays.copyOfRange(args, 10, args.length)); + boolean parallel = "true".equals(args[10]); + Set jvmArgs = Sets.newHashSet(Arrays.copyOfRange(args, 11, args.length)); if (jvmArgs.stream().noneMatch(arg -> arg.contains("-Xmx"))) jvmArgs.add("-Xmx5G"); // Load environments - List envs; - if (Files.isDirectory(envsRoot)) { - envs = Files.walk(envsRoot).filter(path -> !Files.isDirectory(path)) + List envs = new ArrayList<>(); + for (Path envPath : envPaths) { + if (Files.isDirectory(envPath)) { + envs.addAll(Files.walk(envPath).filter(path -> !Files.isDirectory(path)) .map(path -> { try { - return gson.fromJson(new String(Files.readAllBytes(path), StandardCharsets.UTF_8), Environment.class); + return gson.fromJson(Files.readString(path), Environment.class); } catch (JsonSyntaxException | IOException e) { throw new RuntimeException(e); } - }).collect(Collectors.toList()); - } else { - envs = Collections.singletonList(gson.fromJson(new String( - Files.readAllBytes(envsRoot),StandardCharsets.UTF_8), Environment.class)); + }).toList()); + } else { + envs.add(gson.fromJson(Files.readString(envPath), Environment.class)); + } } - System.out.println("Test environments: " + String.join(", ", - envs.stream().map(Environment::getName).collect(Collectors.toList()))); + + System.out.println("Test environments: " + envs.stream().map(Environment::getName).collect(Collectors.joining(", "))); Set allTests = new HashSet<>(); - Map>> failures = new HashMap<>(); + Map> failures = new HashMap<>(); boolean docsFailed = false; // Run tests and collect the results envs.sort(Comparator.comparing(Environment::getName)); + int parallelPort = 25600; // Port to start from for parallel tests, so the servers don't clash + long finalTimeout = timeout; + List tasks = new ArrayList<>(); + Map collectedResults = Collections.synchronizedMap(new HashMap<>()); for (Environment env : envs) { - System.out.println("Starting testing on " + env.getName()); - env.initialize(dataRoot, runnerRoot, false); - TestResults results = env.runTests(runnerRoot, testsRoot, devMode, genDocs, jUnit, debug, verbosity, timeout, jvmArgs); - if (results == null) { - if (devMode) { - // Nothing to report, it's the dev mode environment. - System.exit(0); - return; + int finalParallelPort = parallelPort; + tasks.add(() -> { + try { + System.out.println("Starting testing on " + env.getName()); + env.initialize(dataRoot, runnerRoot, false, finalParallelPort); + TestResults results = env.runTests(runnerRoot, testsRoot, devMode, genDocs, jUnit, debug, verbosity, finalTimeout, jvmArgs); + if (results == null) { + if (devMode) { + // Nothing to report, it's the dev mode environment. + System.exit(0); + return; + } + System.err.println("The test environment '" + env.getName() + "' failed to produce test results."); + System.exit(3); + return; + } + collectedResults.put(env, results); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); } - System.err.println("The test environment '" + env.getName() + "' failed to produce test results."); - System.exit(3); - return; + }); + if (parallel) { + parallelPort += 1; // Increment port for next environment } - - // Collect results - docsFailed = results.docsFailed(); + } + + // Execute tasks either in parallel or sequentially + if (parallel) { + List> futures = new ArrayList<>(); + for (Runnable task : tasks) { + futures.add(CompletableFuture.runAsync(task)); + } + // Wait for all to complete + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } else { + for (Runnable task : tasks) { + task.run(); + } + } + + // Process collected results + for (var entry : collectedResults.entrySet()) { + TestResults results = entry.getValue(); + Environment env = entry.getKey(); + docsFailed |= results.docsFailed(); allTests.addAll(results.getSucceeded()); allTests.addAll(results.getFailed().keySet()); for (Map.Entry fail : results.getFailed().entrySet()) { String error = fail.getValue(); assert error != null; failures.computeIfAbsent(fail.getKey(), (k) -> new ArrayList<>()) - .add(new NonNullPair<>(env, error)); + .add(new TestError(env, error)); } } @@ -119,33 +147,33 @@ public static void main(String... args) throws IOException, InterruptedException } // Sort results in alphabetical order - List succeeded = allTests.stream().filter(name -> !failures.containsKey(name)).collect(Collectors.toList()); - Collections.sort(succeeded); + List succeeded = allTests.stream().filter(name -> !failures.containsKey(name)).sorted().collect(Collectors.toList()); List failNames = new ArrayList<>(failures.keySet()); Collections.sort(failNames); // All succeeded tests in a single line StringBuilder output = new StringBuilder(String.format("%s Results %s%n", StringUtils.repeat("-", 25), StringUtils.repeat("-", 25))); - output.append("\nTested environments: " + String.join(", ", - envs.stream().map(Environment::getName).collect(Collectors.toList()))); - output.append("\nSucceeded:\n " + String.join((jUnit ? "\n " : ", "), succeeded)); + output.append("\nTested environments: ").append(envs.stream().map(Environment::getName).collect(Collectors.joining(", "))); + output.append("\nSucceeded:\n ").append(String.join((jUnit ? "\n " : ", "), succeeded)); if (!failNames.isEmpty()) { // More space for failed tests, they're important output.append("\nFailed:"); for (String failed : failNames) { - List> errors = failures.get(failed); - output.append("\n " + failed + " (on " + errors.size() + " environment" + (errors.size() == 1 ? "" : "s") + ")"); - for (NonNullPair error : errors) { - output.append("\n " + error.getSecond() + " (on " + error.getFirst().getName() + ")"); + List errors = failures.get(failed); + output.append("\n ").append(failed).append(" (on ").append(errors.size()).append(" environment").append(errors.size() == 1 ? "" : "s").append(")"); + for (TestError error : errors) { + output.append("\n ").append(error.message()).append(" (on ").append(error.environment().getName()).append(")"); } } output.append(String.format("%n%n%s", StringUtils.repeat("-", 60))); - System.err.print(output.toString()); + System.err.print(output); System.exit(failNames.size()); // Error code to indicate how many tests failed. return; } output.append(String.format("%n%n%s", StringUtils.repeat("-", 60))); - System.out.print(output.toString()); + System.out.print(output); } + private record TestError(Environment environment, String message) { } + } diff --git a/src/test/resources/runner_data/server.properties.generic b/src/test/resources/runner_data/server.properties.generic index 84816088740..a32881f64d2 100644 --- a/src/test/resources/runner_data/server.properties.generic +++ b/src/test/resources/runner_data/server.properties.generic @@ -42,4 +42,4 @@ resource-pack-sha1= spawn-protection=0 online-mode=true allow-flight=false -max-world-size=29999984 \ No newline at end of file +max-world-size=29999984