diff --git a/Dockerfile b/Dockerfile
index 3c9fab7..1b6de8b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -63,6 +63,7 @@ FROM eclipse-temurin:11-jre-focal
WORKDIR /opt/release-download-directory
COPY --from=build-download-directory /gitroot/reactome-release-directory/target/download-directory.jar target/
+COPY --from=build-download-directory /gitroot/reactome-release-directory/target/download-directory-verifier.jar target/
COPY --from=build-download-directory /gitroot/reactome-release-directory/src/main/resources/ src/main/resources/
diff --git a/Jenkinsfile b/Jenkinsfile
index f0f6c3e..edca522 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -7,13 +7,13 @@ import org.reactome.release.jenkins.utilities.Utilities
def utils = new Utilities()
pipeline {
- agent any
+ agent any
- environment {
- ECR_URL = 'public.ecr.aws/reactome/release-download-directory'
- CONT_NAME = 'release_download_directory_container'
- CONT_ROOT = '/opt/release-download-directory'
- }
+ environment {
+ ECR_URL = 'public.ecr.aws/reactome/release-download-directory'
+ CONT_NAME = 'release_download_directory_container'
+ CONT_ROOT = '/opt/release-download-directory'
+ }
stages {
// This stage checks that an upstream project 'BioModels' was run successfully for its last build.
@@ -47,7 +47,17 @@ pipeline {
sh "sudo service neo4j stop"
sh "mkdir -p config"
sh "sudo cp $ConfigFile config/auth.properties"
- sh "docker run -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock -v \$(pwd)/config:/config -v \$(pwd)/${releaseVersion}:/opt/release-download-directory/${releaseVersion} --net=host ${ECR_URL}:latest /bin/bash -c \'java -Xmx${env.JAVA_MEM_MAX}m -javaagent:src/main/resources/spring-instrument-4.2.4.RELEASE.jar -jar target/download-directory.jar -g /config/auth.properties\'"
+ sh """
+ docker run \\
+ --rm \\
+ -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock \\
+ -v ${pwd()}/config:/config \\
+ -v ${pwd()}/${releaseVersion}:${CONT_ROOT}/${releaseVersion} \\
+ --net=host \\
+ --name ${CONT_NAME} \\
+ ${ECR_URL}:latest \\
+ /bin/bash -c "java -Xmx${env.JAVA_MEM_MAX}m -javaagent:src/main/resources/spring-instrument-4.2.4.RELEASE.jar -jar target/download-directory.jar -g /config/auth.properties"
+ """
sh "sudo service neo4j start"
sh "sudo service tomcat9 start"
sh "sudo mv ${releaseVersion}/* ${env.ABS_DOWNLOAD_PATH}/${releaseVersion}/"
@@ -56,45 +66,80 @@ pipeline {
}
}
}
-
+
+ // Execute the verifier jar file checking for the existence and proper file sizes of the download_directory output
+ stage('Post: Verify DownloadDirectory ran correctly') {
+ steps {
+ script {
+ def releaseVersion = utils.getReleaseVersion()
+
+ sh """
+ docker run \\
+ --rm \\
+ -v ${pwd()}/${releaseVersion}:${CONT_ROOT}/${releaseVersion}/ \\
+ -v \$HOME/.aws:/root/.aws:ro \\
+ -e AWS_REGION=us-east-1 \\
+ --net=host \\
+ --name ${CONT_NAME}_verifier \\
+ ${ECR_URL}:latest \\
+ /bin/bash -c "java -jar target/data-export-verifier.jar --releaseNumber ${releaseVersion} --output ${CONT_ROOT}/${releaseVersion}"
+ """
+ }
+ }
+ }
+
+ // Creates a list of files and their sizes to use for comparison baseline during next release
+ stage('Post: Create files and sizes list to upload for next release\'s verifier') {
+ steps {
+ script {
+ def fileSizeList = "files_and_sizes.txt"
+ def releaseVersion = utils.getReleaseVersion()
+
+ sh "find ${releaseNumber} -type f -printf \"%s\t%P\n\" > ${fileSizeList}"
+ sh "aws s3 --no-progress cp ${fileSizeList} s3://reactome/private/releases/${releaseVersion}/download_directory/data/"
+ sh "rm ${fileSizeList}"
+ }
+ }
+ }
+
stage('Post: Validate Release Number in BioPax Files') {
- steps {
- script {
- def biopaxSandboxDir = 'biopax/'
- def releaseVersion = utils.getReleaseVersion()
+ steps {
+ script {
+ def biopaxSandboxDir = 'biopax/'
+ def releaseVersion = utils.getReleaseVersion()
- sh """
- set -e
+ sh """
+ set -e
- mkdir -p ${biopaxSandboxDir}
- cp ${env.ABS_DOWNLOAD_PATH}/${releaseVersion}/biopax* ${biopaxSandboxDir}
+ mkdir -p ${biopaxSandboxDir}
+ cp ${env.ABS_DOWNLOAD_PATH}/${releaseVersion}/biopax* ${biopaxSandboxDir}
- cd ${biopaxSandboxDir}
+ cd ${biopaxSandboxDir}
- unzip -o biopax.zip
+ unzip -o biopax.zip
- numOfOwlFiles=\$(ls -1 *.owl | wc -l)
- numOfCorrectOwlFiles=\$(grep -l "xml:base=\\"http://www.reactome.org/biopax/${releaseVersion}" *.owl | wc -l)
+ numOfOwlFiles=\$(ls -1 *.owl | wc -l)
+ numOfCorrectOwlFiles=\$(grep -l "xml:base=\\"http://www.reactome.org/biopax/${releaseVersion}" *.owl | wc -l)
- echo "Found \$numOfOwlFiles OWL files, \$numOfCorrectOwlFiles have correct version"
+ echo "Found \$numOfOwlFiles OWL files, \$numOfCorrectOwlFiles have correct version"
- if [ "\$numOfOwlFiles" -eq "\$numOfCorrectOwlFiles" ]; then
- echo "BioPax 3 files have correct Reactome version"
- else
- echo "Not all BioPax 3 files have correct Reactome version."
- exit 1
- fi
- """
- }
- }
+ if [ "\$numOfOwlFiles" -eq "\$numOfCorrectOwlFiles" ]; then
+ echo "BioPax 3 files have correct Reactome version"
+ else
+ echo "Not all BioPax 3 files have correct Reactome version."
+ exit 1
+ fi
+ """
+ }
+ }
- post {
- always {
- sh "rm -rf biopax/"
- }
- }
+ post {
+ always {
+ sh "rm -rf biopax/"
+ }
+ }
}
-
+
// This stage archives all logs and other outputs produced by DownloadDirectory on S3.
stage('Post: Archive logs and validation files'){
steps{
diff --git a/pom.xml b/pom.xml
index d0af103..ffc27cc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -182,6 +182,11 @@
${spring.version}
-->
+
+ org.reactome.release.verifier
+ verifier-lib
+ 1.0-SNAPSHOT
+
@@ -260,30 +265,39 @@
org.apache.maven.plugins
maven-shade-plugin
3.1.0
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+ META-INF/spring.handlers
+
+
+ META-INF/spring.schemas
+
+
+
+
+ download-directory
package
shade
-
-
- *:*
-
- META-INF/*.SF
- META-INF/*.DSA
- META-INF/*.RSA
-
-
-
-
-
- META-INF/spring.handlers
-
-
- META-INF/spring.schemas
-
+ download-directory
+
org.reactome.release.downloaddirectory.Main
@@ -291,8 +305,25 @@
- download-directory
+
+
+
+
+ download-directory-verifier
+ package
+
+ shade
+
+
+ download-directory-verifier
+
+
+
+ org.reactome.release.downloaddirectory.verifier.DownloadDirectoryVerifier
+
+
+
diff --git a/src/main/java/org/reactome/release/downloaddirectory/verifier/DownloadDirectoryVerifier.java b/src/main/java/org/reactome/release/downloaddirectory/verifier/DownloadDirectoryVerifier.java
new file mode 100644
index 0000000..a3d58e4
--- /dev/null
+++ b/src/main/java/org/reactome/release/downloaddirectory/verifier/DownloadDirectoryVerifier.java
@@ -0,0 +1,19 @@
+package org.reactome.release.downloaddirectory.verifier;
+
+import org.reactome.release.verifier.DefaultVerifier;
+import org.reactome.release.verifier.Verifier;
+
+import java.io.IOException;
+
+/**
+ * @author Joel Weiser (joel.weiser@oicr.on.ca)
+ * Created 12/20/2025
+ */
+public class DownloadDirectoryVerifier {
+
+ public static void main(String[] args) throws IOException {
+ Verifier verifier = new DefaultVerifier("download_directory");
+ verifier.parseCommandLineArgs(args);
+ verifier.run();
+ }
+}