Feature/verifier

Dockerfile

@@ -24,15 +24,15 @@ CMD mvn -B -q checkstyle:check | \
 # ===== stage 2 =====
 FROM setup-env AS build-jar
 
-RUN mvn clean compile assembly:single
+RUN mvn clean package -DskipTests
 
 
 # ===== stage 3 =====
 FROM eclipse-temurin:11-jre-focal
 
 ARG REPO_DIR
 
-ARG JAR_FILE=target/orthopairs-*-jar-with-dependencies.jar
+ARG JAR_FILE=target/orthopairs-jar-with-dependencies.jar
 
 WORKDIR ${REPO_DIR}
 

Jenkinsfile

@@ -48,7 +48,7 @@ pipeline{
 		stage('Setup: Build jar file') {
 			steps {
 				script {
-					utils.buildJarFile()
+					sh "mvn clean package -DskipTests"
 				}
 			}
 		}
@@ -58,27 +58,35 @@ pipeline{
 				script {
 					// The credentials used here are a config file uploaded to Jenkins.
 					withCredentials([file(credentialsId: 'Config', variable: 'ConfigFile')]) {
-						sh "java -jar target/orthopairs-*-jar-with-dependencies.jar $ConfigFile"
+						sh "java -jar target/orthopairs-jar-with-dependencies.jar $ConfigFile"
 					}
 				}
 			}
 		}
-		// This stage compares the line counts of the orthopairs files generated between the current and previous release.
-		// An intelligible output should be visible at the console logs for the build.
-		stage('Post: Orthopairs file line counts') {
-		    steps {
-		        script {
-		            def releaseVersion = utils.getReleaseVersion()
-		            def previousReleaseVersion = utils.getPreviousReleaseVersion()
-		            def currentDir = pwd()
-		            sh "mkdir -p ${previousReleaseVersion}/"
-		            sh "aws s3 --recursive --no-progress cp s3://reactome/private/releases/${previousReleaseVersion}/orthopairs/data/orthopairs/ ${previousReleaseVersion}/"
-		            sh "gunzip -q ${previousReleaseVersion}/*"
-		            utils.outputLineCountsOfFilesBetweenFolders("$releaseVersion", "$previousReleaseVersion", "$currentDir")
-		            sh "rm -r ${previousReleaseVersion}"
-		        }
-		    }
+
+
+		// This stage verifies orthopairs have correct size by comparing against the previous releases' files
+		stage('Post: Verify Orthopairs files have correct size') {
+			steps {
+				script {
+					def currentRelease = (pwd() =~ /Releases\/(\d+)\//)[0][1];
+					try {
+						sh "java -jar target/orthopairs-verifier-jar-with-dependencies.jar -release $currentRelease"
+					} catch (Exception e) {
+						def userInput = input(
+							id: 'userInput', message: "Orthopairs has errors - do you want to continue the orthopair pipeline",
+							parameters: [
+								[$class: 'BooleanParameterDefinition', defaultValue: true, name: 'response']
+							]
+						)
+						if (!userInput) {
+							error('Please manually address the orthopair error messages before continuing');
+						}
+					}
+				}
+			}
 		}
+
 		// Logs and data files generated by this step are archived in the Reactome S3 bucket.
 		// All files are then deleted on server.
 		stage('Post: Archive Outputs'){

checkstyle.xml

@@ -3,7 +3,7 @@
         "https://checkstyle.org/dtds/configuration_1_3.dtd">
 <module name="Checker">
     <module name="LineLength">
-        <property name="max" value="150"/>
+        <property name="max" value="250"/>
     </module>
     <!-- Add more modules as needed -->
 </module>

pom.xml

@@ -61,9 +61,27 @@
 		<dependency>
 			<groupId>org.apache.logging.log4j</groupId>
 			<artifactId>log4j-core</artifactId>
-			<version>2.11.0</version>
+			<version>2.17.0</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-api</artifactId>
+			<version>2.17.0</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.jcommander</groupId>
+			<artifactId>jcommander</artifactId>
+			<version>1.83</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.reactome.release.verifier</groupId>
+			<artifactId>verifier-lib</artifactId>
+			<version>1.0-SNAPSHOT</version>
 		</dependency>
 
+
 		<!-- Test dependencies -->
 		<dependency>
 			<groupId>junit</groupId>
@@ -97,6 +115,18 @@
 	</dependencies>
 
 	<repositories>
+		<!-- Sonatype repository from which to obtain SNAPSHOT dependencies created by Reactome (or others sources) -->
+		<repository>
+			<id>oss.sonatype.org-snapshot</id>
+			<url>http://oss.sonatype.org/content/repositories/snapshots</url>
+			<releases>
+				<enabled>false</enabled>
+			</releases>
+			<snapshots>
+				<enabled>true</enabled>
+			</snapshots>
+		</repository>
+
 		<repository>
 			<id>nexus-ebi-repo</id>
 			<name>The EBI internal repository</name>
@@ -132,24 +162,46 @@
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-assembly-plugin</artifactId>
 				<version>3.2.0</version>
-				<configuration>
-					<archive>
-						<manifestEntries>
-							<Multi-Release>true</Multi-Release>
-							<Main-Class>org.reactome.release.orthopairs.Main</Main-Class>
-						</manifestEntries>
-					</archive>
-					<descriptorRefs>
-						<descriptorRef>jar-with-dependencies</descriptorRef>
-					</descriptorRefs>
-				</configuration>
 				<executions>
 					<execution>
-						<id>make-assembly</id> <!-- this is used for inheritance merges -->
+						<id>main-build</id>
+						<phase>package</phase>
+						<goals>
+							<goal>single</goal>
+						</goals>
+
+						<configuration>
+							<archive>
+								<manifestEntries>
+									<Multi-Release>true</Multi-Release>
+									<Main-Class>org.reactome.release.orthopairs.Main</Main-Class>
+								</manifestEntries>
+							</archive>
+							<descriptorRefs>
+								<descriptorRef>jar-with-dependencies</descriptorRef>
+							</descriptorRefs>
+							<finalName>orthopairs</finalName>
+						</configuration>
+					</execution>
+					<execution>
+						<id>verifier-build</id> <!-- this is used for inheritance merges -->
 						<phase>package</phase> <!-- bind to the packaging phase -->
 						<goals>
 							<goal>single</goal>
 						</goals>
+
+						<configuration>
+							<archive>
+								<manifestEntries>
+									<Multi-Release>true</Multi-Release>
+									<Main-Class>org.reactome.release.orthopairs.verifier.Verifier</Main-Class>
+								</manifestEntries>
+							</archive>
+							<descriptorRefs>
+								<descriptorRef>jar-with-dependencies</descriptorRef>
+							</descriptorRefs>
+							<finalName>orthopairs-verifier</finalName>
+						</configuration>
 					</execution>
 				</executions>
 			</plugin>

src/main/java/org/reactome/release/orthopairs/verifier/Verifier.java

@@ -0,0 +1,178 @@
+package org.reactome.release.orthopairs.verifier;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+import org.reactome.release.verifier.Results;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.reactome.release.verifier.CountUtils.greaterThanOrEqualTo5PercentDrop;
+import static org.reactome.release.verifier.CountUtils.greaterThanOrEqualToXPercentDrop;
+import static org.reactome.release.verifier.FileUtils.*;
+
+/**
+ * @author Joel Weiser (joel.weiser@oicr.on.ca)
+ * Created 8/15/2024
+ */
+public class Verifier {
+    @Parameter(names = "-release", description = "Current Reactome Release number", required = true)
+    private int releaseNumber;
+
+    public static void main(String[] args) throws IOException {
+        Verifier verifier = new Verifier();
+        JCommander.newBuilder()
+            .addObject(verifier)
+            .build()
+            .parse(args);
+
+        verifier.run();
+    }
+
+    private void run() throws IOException {
+        Results results = compareCurrentAndPreviousOrthopairFiles();
+
+        results.reportInfoMessages();
+        if (results.hasErrors()) {
+            results.reportErrors();
+            System.exit(1);
+        }
+    }
+
+    private Results compareCurrentAndPreviousOrthopairFiles() throws IOException {
+        Results overallResults = new Results();
+
+        List<Path> previousOrthopairFilePaths = getPreviousOrthopairFilePaths();
+        for (Path currentOrthopairFilePath : getCurrentOrthopairFilePaths()) {
+            Path previousOrthopairFilePath =
+                getAnalogousPreviousOrthopairFile(currentOrthopairFilePath, previousOrthopairFilePaths);
+
+            if (previousOrthopairFilePath != null) {
+                Results lineCountResults = checkLineCount(currentOrthopairFilePath, previousOrthopairFilePath);
+                Results fileSizeResults = checkFileSize(currentOrthopairFilePath, previousOrthopairFilePath);
+
+                overallResults.addInfoMessages(lineCountResults.getInfoMessages());
+                overallResults.addInfoMessages(fileSizeResults.getInfoMessages());
+                overallResults.addErrorMessages(lineCountResults.getErrorMessages());
+                overallResults.addErrorMessages(fileSizeResults.getErrorMessages());
+            }
+        }
+
+        return overallResults;
+    }
+
+    private List<Path> getCurrentOrthopairFilePaths() throws IOException {
+        return Files.list(Paths.get(String.valueOf(releaseNumber))).collect(Collectors.toList());
+    }
+
+    private List<Path> getPreviousOrthopairFilePaths() throws IOException {
+        String folderName = downloadPreviousOrthopairFiles();
+        Files.list(Paths.get(folderName)).filter(file -> file.getFileName().toString().endsWith(".gz")).forEach(file -> {
+            try {
+                gunzipFile(file);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        return Files.list(Paths.get(folderName)).filter(file -> file.getFileName().toString().endsWith(".tsv")).collect(Collectors.toList());
+    }
+
+    private Path getAnalogousPreviousOrthopairFile(
+        Path currentOrthopairFilePath, List<Path> previousOrthopairFilePaths) {
+
+        return previousOrthopairFilePaths
+            .stream()
+            .filter(previousOrthopairFilePath ->
+                previousOrthopairFilePath.getFileName().toString()
+                    .equals(currentOrthopairFilePath.getFileName().toString())
+            )
+            .findFirst()
+            .orElse(null);
+    }
+
+    private Results checkLineCount(Path currentOrthopairFilePath, Path previousOrthopairFilePath)
+        throws IOException {
+
+        Results lineCountResults = new Results();
+
+        int currentLineCount = (int) Files.lines(currentOrthopairFilePath).count();
+        int previousLineCount = (int) Files.lines(previousOrthopairFilePath).count();
+
+        if (greaterThanOrEqualToXPercentDrop(currentLineCount, previousLineCount, 10)) {
+            lineCountResults.addErrorMessage(
+                String.format("%s has significantly fewer lines than %s (%d vs. %d) - Difference: %d lines (%.2f%%)",
+                    currentOrthopairFilePath, previousOrthopairFilePath, currentLineCount, previousLineCount,
+                    (previousLineCount - currentLineCount),
+                    ((float) (previousLineCount - currentLineCount) / previousLineCount) * 100.0
+                )
+            );
+        } else {
+            lineCountResults.addInfoMessage(
+                String.format("%s (%d lines) vs. %s (%d lines) - Difference: %d lines (%.2f%%)",
+                    currentOrthopairFilePath, currentLineCount, previousOrthopairFilePath, previousLineCount,
+                    (currentLineCount - previousLineCount),
+                    ((float) (previousLineCount - currentLineCount) / previousLineCount) * 100.0
+                )
+            );
+        }
+
+        return lineCountResults;
+    }
+
+    private Results checkFileSize(Path currentOrthopairFilePath, Path previousOrthopairFilePath) throws IOException {
+
+        Results fileSizeResults = new Results();
+
+        int currentFileSize = (int) Files.size(currentOrthopairFilePath);
+        int previousFileSize = (int) Files.size(previousOrthopairFilePath);
+
+        if (greaterThanOrEqualToXPercentDrop(currentFileSize, previousFileSize, 10)) {
+            fileSizeResults.addErrorMessage(
+                String.format("%s has significantly smaller size than %s (%d bytes vs. %d bytes) - Difference: %d bytes (%.2f%%)",
+                    currentOrthopairFilePath, previousOrthopairFilePath, currentFileSize, previousFileSize,
+                    (previousFileSize - currentFileSize),
+                    ((float) (previousFileSize - currentFileSize) / previousFileSize) * 100.0
+                )
+            );
+        } else {
+            fileSizeResults.addInfoMessage(
+                String.format("%s (%d bytes) vs. %s (%d bytes) - Difference: %d bytes (%.2f%%)",
+                    currentOrthopairFilePath, currentFileSize, previousOrthopairFilePath, previousFileSize,
+                    (currentFileSize - previousFileSize),
+                    ((float) (currentFileSize - previousFileSize) / previousFileSize) * 100.0
+                )
+            );
+        }
+
+        return fileSizeResults;
+    }
+
+
+    private String downloadPreviousOrthopairFiles() throws IOException {
+        deleteDirectory(getPreviousReleaseNumberDirectory());
+        downloadFolderFromS3("reactome", getS3FolderPath());
+        return renameFolderToPreviousReleaseVersionNumber().toString();
+    }
+
+    private Path renameFolderToPreviousReleaseVersionNumber() throws IOException {
+        Files.move(Paths.get(getS3FolderPath()).getFileName(), getPreviousReleaseNumberDirectory());
+        return getPreviousReleaseNumberDirectory();
+    }
+
+    private String getS3FolderPath() {
+        return String.format("private/releases/%d/orthopairs/data/orthopairs/", getPreviousReleaseNumber());
+    }
+
+    private Path getPreviousReleaseNumberDirectory() {
+        return Paths.get(String.valueOf(getPreviousReleaseNumber()));
+    }
+
+    private int getPreviousReleaseNumber() {
+        return this.releaseNumber - 1;
+    }
+}