diff --git a/Dockerfile b/Dockerfile
index 2c0f414..08ef07f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,7 @@ CMD mvn -B -q checkstyle:check | \
# ===== stage 2 =====
FROM setup-env AS build-jar
-RUN mvn clean compile assembly:single
+RUN mvn clean package -DskipTests
# ===== stage 3 =====
@@ -32,7 +32,7 @@ FROM eclipse-temurin:11-jre-focal
ARG REPO_DIR
-ARG JAR_FILE=target/orthopairs-*-jar-with-dependencies.jar
+ARG JAR_FILE=target/orthopairs-jar-with-dependencies.jar
WORKDIR ${REPO_DIR}
diff --git a/Jenkinsfile b/Jenkinsfile
index 8a466e7..1f243de 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -48,7 +48,7 @@ pipeline{
stage('Setup: Build jar file') {
steps {
script {
- utils.buildJarFile()
+ sh "mvn clean package -DskipTests"
}
}
}
@@ -58,27 +58,35 @@ pipeline{
script {
// The credentials used here are a config file uploaded to Jenkins.
withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
- sh "java -jar target/orthopairs-*-jar-with-dependencies.jar $ConfigFile"
+ sh "java -jar target/orthopairs-jar-with-dependencies.jar $ConfigFile"
}
}
}
}
- // This stage compares the line counts of the orthopairs files generated between the current and previous release.
- // An intelligible output should be visible at the console logs for the build.
- stage('Post: Orthopairs file line counts') {
- steps {
- script {
- def releaseVersion = utils.getReleaseVersion()
- def previousReleaseVersion = utils.getPreviousReleaseVersion()
- def currentDir = pwd()
- sh "mkdir -p ${previousReleaseVersion}/"
- sh "aws s3 --recursive --no-progress cp s3://reactome/private/releases/${previousReleaseVersion}/orthopairs/data/orthopairs/ ${previousReleaseVersion}/"
- sh "gunzip -q ${previousReleaseVersion}/*"
- utils.outputLineCountsOfFilesBetweenFolders("$releaseVersion", "$previousReleaseVersion", "$currentDir")
- sh "rm -r ${previousReleaseVersion}"
- }
- }
+
+
+ // This stage verifies orthopairs have correct size by comparing against the previous releases' files
+ stage('Post: Verify Orthopairs files have correct size') {
+ steps {
+ script {
+ def currentRelease = (pwd() =~ /Releases\/(\d+)\//)[0][1];
+ try {
+ sh "java -jar target/orthopairs-verifier-jar-with-dependencies.jar -release $currentRelease"
+ } catch (Exception e) {
+ def userInput = input(
+ id: 'userInput', message: "Orthopairs has errors - do you want to continue the orthopair pipeline",
+ parameters: [
+ [$class: 'BooleanParameterDefinition', defaultValue: true, name: 'response']
+ ]
+ )
+ if (!userInput) {
+ error('Please manually address the orthopair error messages before continuing');
+ }
+ }
+ }
+ }
}
+
// Logs and data files generated by this step are archived in the Reactome S3 bucket.
// All files are then deleted on server.
stage('Post: Archive Outputs'){
diff --git a/checkstyle.xml b/checkstyle.xml
index 8506a80..c9d5dc6 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -3,7 +3,7 @@
"https://checkstyle.org/dtds/configuration_1_3.dtd">
-
+
diff --git a/pom.xml b/pom.xml
index 2a40667..b85b55a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,9 +61,27 @@
org.apache.logging.log4j
log4j-core
- 2.11.0
+ 2.17.0
+
+
+ org.apache.logging.log4j
+ log4j-api
+ 2.17.0
+
+
+
+ org.jcommander
+ jcommander
+ 1.83
+
+
+
+ org.reactome.release.verifier
+ verifier-lib
+ 1.0-SNAPSHOT
+
junit
@@ -97,6 +115,18 @@
+
+
+ oss.sonatype.org-snapshot
+ http://oss.sonatype.org/content/repositories/snapshots
+
+ false
+
+
+ true
+
+
+
nexus-ebi-repo
The EBI internal repository
@@ -132,24 +162,46 @@
org.apache.maven.plugins
maven-assembly-plugin
3.2.0
-
-
-
- true
- org.reactome.release.orthopairs.Main
-
-
-
- jar-with-dependencies
-
-
- make-assembly
+ main-build
+ package
+
+ single
+
+
+
+
+
+ true
+ org.reactome.release.orthopairs.Main
+
+
+
+ jar-with-dependencies
+
+ orthopairs
+
+
+
+ verifier-build
package
single
+
+
+
+
+ true
+ org.reactome.release.orthopairs.verifier.Verifier
+
+
+
+ jar-with-dependencies
+
+ orthopairs-verifier
+
diff --git a/src/main/java/org/reactome/release/orthopairs/verifier/Verifier.java b/src/main/java/org/reactome/release/orthopairs/verifier/Verifier.java
new file mode 100644
index 0000000..f1049ce
--- /dev/null
+++ b/src/main/java/org/reactome/release/orthopairs/verifier/Verifier.java
@@ -0,0 +1,178 @@
+package org.reactome.release.orthopairs.verifier;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+import org.reactome.release.verifier.Results;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.reactome.release.verifier.CountUtils.greaterThanOrEqualTo5PercentDrop;
+import static org.reactome.release.verifier.CountUtils.greaterThanOrEqualToXPercentDrop;
+import static org.reactome.release.verifier.FileUtils.*;
+
+/**
+ * @author Joel Weiser (joel.weiser@oicr.on.ca)
+ * Created 8/15/2024
+ */
+public class Verifier {
+ @Parameter(names = "-release", description = "Current Reactome Release number", required = true)
+ private int releaseNumber;
+
+ public static void main(String[] args) throws IOException {
+ Verifier verifier = new Verifier();
+ JCommander.newBuilder()
+ .addObject(verifier)
+ .build()
+ .parse(args);
+
+ verifier.run();
+ }
+
+ private void run() throws IOException {
+ Results results = compareCurrentAndPreviousOrthopairFiles();
+
+ results.reportInfoMessages();
+ if (results.hasErrors()) {
+ results.reportErrors();
+ System.exit(1);
+ }
+ }
+
+ private Results compareCurrentAndPreviousOrthopairFiles() throws IOException {
+ Results overallResults = new Results();
+
+ List previousOrthopairFilePaths = getPreviousOrthopairFilePaths();
+ for (Path currentOrthopairFilePath : getCurrentOrthopairFilePaths()) {
+ Path previousOrthopairFilePath =
+ getAnalogousPreviousOrthopairFile(currentOrthopairFilePath, previousOrthopairFilePaths);
+
+ if (previousOrthopairFilePath != null) {
+ Results lineCountResults = checkLineCount(currentOrthopairFilePath, previousOrthopairFilePath);
+ Results fileSizeResults = checkFileSize(currentOrthopairFilePath, previousOrthopairFilePath);
+
+ overallResults.addInfoMessages(lineCountResults.getInfoMessages());
+ overallResults.addInfoMessages(fileSizeResults.getInfoMessages());
+ overallResults.addErrorMessages(lineCountResults.getErrorMessages());
+ overallResults.addErrorMessages(fileSizeResults.getErrorMessages());
+ }
+ }
+
+ return overallResults;
+ }
+
+ private List getCurrentOrthopairFilePaths() throws IOException {
+ return Files.list(Paths.get(String.valueOf(releaseNumber))).collect(Collectors.toList());
+ }
+
+ private List getPreviousOrthopairFilePaths() throws IOException {
+ String folderName = downloadPreviousOrthopairFiles();
+ Files.list(Paths.get(folderName)).filter(file -> file.getFileName().toString().endsWith(".gz")).forEach(file -> {
+ try {
+ gunzipFile(file);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ return Files.list(Paths.get(folderName)).filter(file -> file.getFileName().toString().endsWith(".tsv")).collect(Collectors.toList());
+ }
+
+ private Path getAnalogousPreviousOrthopairFile(
+ Path currentOrthopairFilePath, List previousOrthopairFilePaths) {
+
+ return previousOrthopairFilePaths
+ .stream()
+ .filter(previousOrthopairFilePath ->
+ previousOrthopairFilePath.getFileName().toString()
+ .equals(currentOrthopairFilePath.getFileName().toString())
+ )
+ .findFirst()
+ .orElse(null);
+ }
+
+ private Results checkLineCount(Path currentOrthopairFilePath, Path previousOrthopairFilePath)
+ throws IOException {
+
+ Results lineCountResults = new Results();
+
+ int currentLineCount = (int) Files.lines(currentOrthopairFilePath).count();
+ int previousLineCount = (int) Files.lines(previousOrthopairFilePath).count();
+
+ if (greaterThanOrEqualToXPercentDrop(currentLineCount, previousLineCount, 10)) {
+ lineCountResults.addErrorMessage(
+ String.format("%s has significantly fewer lines than %s (%d vs. %d) - Difference: %d lines (%.2f%%)",
+ currentOrthopairFilePath, previousOrthopairFilePath, currentLineCount, previousLineCount,
+ (previousLineCount - currentLineCount),
+ ((float) (previousLineCount - currentLineCount) / previousLineCount) * 100.0
+ )
+ );
+ } else {
+ lineCountResults.addInfoMessage(
+ String.format("%s (%d lines) vs. %s (%d lines) - Difference: %d lines (%.2f%%)",
+ currentOrthopairFilePath, currentLineCount, previousOrthopairFilePath, previousLineCount,
+ (currentLineCount - previousLineCount),
+ ((float) (previousLineCount - currentLineCount) / previousLineCount) * 100.0
+ )
+ );
+ }
+
+ return lineCountResults;
+ }
+
+ private Results checkFileSize(Path currentOrthopairFilePath, Path previousOrthopairFilePath) throws IOException {
+
+ Results fileSizeResults = new Results();
+
+ int currentFileSize = (int) Files.size(currentOrthopairFilePath);
+ int previousFileSize = (int) Files.size(previousOrthopairFilePath);
+
+ if (greaterThanOrEqualToXPercentDrop(currentFileSize, previousFileSize, 10)) {
+ fileSizeResults.addErrorMessage(
+ String.format("%s has significantly smaller size than %s (%d bytes vs. %d bytes) - Difference: %d bytes (%.2f%%)",
+ currentOrthopairFilePath, previousOrthopairFilePath, currentFileSize, previousFileSize,
+ (previousFileSize - currentFileSize),
+ ((float) (previousFileSize - currentFileSize) / previousFileSize) * 100.0
+ )
+ );
+ } else {
+ fileSizeResults.addInfoMessage(
+ String.format("%s (%d bytes) vs. %s (%d bytes) - Difference: %d bytes (%.2f%%)",
+ currentOrthopairFilePath, currentFileSize, previousOrthopairFilePath, previousFileSize,
+ (currentFileSize - previousFileSize),
+ ((float) (currentFileSize - previousFileSize) / previousFileSize) * 100.0
+ )
+ );
+ }
+
+ return fileSizeResults;
+ }
+
+
+ private String downloadPreviousOrthopairFiles() throws IOException {
+ deleteDirectory(getPreviousReleaseNumberDirectory());
+ downloadFolderFromS3("reactome", getS3FolderPath());
+ return renameFolderToPreviousReleaseVersionNumber().toString();
+ }
+
+ private Path renameFolderToPreviousReleaseVersionNumber() throws IOException {
+ Files.move(Paths.get(getS3FolderPath()).getFileName(), getPreviousReleaseNumberDirectory());
+ return getPreviousReleaseNumberDirectory();
+ }
+
+ private String getS3FolderPath() {
+ return String.format("private/releases/%d/orthopairs/data/orthopairs/", getPreviousReleaseNumber());
+ }
+
+ private Path getPreviousReleaseNumberDirectory() {
+ return Paths.get(String.valueOf(getPreviousReleaseNumber()));
+ }
+
+ private int getPreviousReleaseNumber() {
+ return this.releaseNumber - 1;
+ }
+}