");
- startDiv = startDiv.concat("An agent developed for performing simulations on HPC servers "
- + "to create a living, continuously self-improving and growing "
- + "Knowledge Graph of species serving the need for thermochemical data across disciplines.");
- startDiv = startDiv.concat("
");
- startDiv = startDiv.concat("
");
- return startDiv;
- }
-
- /**
- * Converts path specific to a Windows or Linux server.
- *
- * @param resourcePath
- * @return
- */
- private String pathToResource(){
- return "/KineticsAgent/assets/";
- }
-
- /**
- * Produces a link to a java script library or style sheet using the
- * link tag and the path where the resource is located.
- *
- * @param link
- * @param hrefStartPath
- * @param hrefEndPath
- * @return
- */
- private String getLink(String link, String hrefStartPath, String hrefEndPath){
- String linkResource = link;
- linkResource = linkResource.concat(hrefStartPath);
- linkResource = linkResource.concat(hrefEndPath);
- return linkResource;
- }
-
- /**
- * Prepares the header of table showing statistics about jobs submitted to DFT Agent.
- *
- * @return mainly the header part of the HTML table showing job statistics.
- */
- public String getStatisticsTableHeader(String headerText, String statisticsProperty, String statisticsValue, String tableWidth){
- String tableHeader = "
"+headerText+"
";
- tableHeader = tableHeader + "
";
- tableHeader = tableHeader + "
";
- tableHeader = tableHeader + "
"+statisticsProperty+"
";
- tableHeader = tableHeader + "
"+statisticsValue+"
";
- tableHeader = tableHeader + "
";
- return tableHeader;
- }
-
- /**
- * Prepares the row of table showing statistics about jobs submitted to DFT Agent.
- *
- * @param property
- * @param value
- * @return
- */
- public String getStatisticsTableRow(String property, String value){
- String tableRow = "
";
- tableRow = tableRow + "
"+property+"
";
- tableRow = tableRow + "
"+value+"
";
- tableRow = tableRow + "
";
- tableRow = tableRow + "
";
- return tableRow;
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobSubmission.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobSubmission.java
deleted file mode 100644
index 8cf17befa..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobSubmission.java
+++ /dev/null
@@ -1,1117 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.json.JSONObject;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-
-import com.jcraft.jsch.Channel;
-import com.jcraft.jsch.ChannelExec;
-import com.jcraft.jsch.ChannelSftp;
-import com.jcraft.jsch.IdentityRepository;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.Session;
-import com.jcraft.jsch.SftpException;
-import com.jcraft.jsch.agentproxy.AgentProxyException;
-import com.jcraft.jsch.agentproxy.Connector;
-import com.jcraft.jsch.agentproxy.RemoteIdentityRepository;
-import com.jcraft.jsch.agentproxy.connector.PageantConnector;
-
-import uk.ac.cam.cares.jps.base.slurm.job.configuration.SlurmJobProperty;
-
-/**
- * This is an API developed for setting-up and running Slurm jobs.
- *
- * @author msff2
- *
- */
-public class JobSubmission{
- private Logger LOGGER = LogManager.getLogger(JobSubmission.class);
- private String hpcAddress;
- private String agentClass;
- private File workspaceDirectory;
- private String workspaceName;
- private String workspaceParentPath;
- boolean isAuthenticated;
-
- public SlurmJobProperty slurmJobProperty = new SlurmJobProperty();
-
- static Session session;
- static JSch jsch = new JSch();
-
- static int scheduledIteration = 0;
- static Set jobsRunning = new HashSet();
-
- public String getHpcAddress() {
- return hpcAddress;
- }
-
- public void setHpcAddress(String hpcAddress) {
- this.hpcAddress = hpcAddress;
- }
-
- public String getAgentClass() {
- return agentClass;
- }
-
- public void setAgentClass(String agentClass) {
- this.agentClass = agentClass;
- }
-
- public File getWorkspaceDirectory() {
- return workspaceDirectory;
- }
-
- public void setWorkspaceDirectory(File workspaceDirectory) {
- this.workspaceDirectory = workspaceDirectory;
- }
-
- public String getWorkspaceName() {
- return workspaceName;
- }
-
- public void setWorkspaceName(String workspaceName) {
- this.workspaceName = workspaceName;
- }
-
- public static void main(String[] args) throws SlurmJobException{
-// JobSubmission jobSubmission = new JobSubmission("DFTAgent", new File(System.getProperty("user.home")), "login-skylake.hpc.cam.ac.uk");
-// jobSubmission.init();
- }
-
- public String getWorkspaceParentPath() {
- return workspaceParentPath;
- }
-
- public void setWorkspaceParentPath(String workspaceParentPath) {
- this.workspaceParentPath = workspaceParentPath;
- }
-
- /**
- * Constructor for this class.
- *
- * @param agentClass the name of the agent class (e.g. DFTAgent and EBRAgent).
- * @param workspaceName the name of workspace
- * will be created, e.g. user home directory read by System.getProperty("user.home");
- * @param hpcAddress the address of HPC, e.g. login-skylake.hpc.cam.ac.uk for CSD3.
- */
- public JobSubmission(String agentClass, String hpcAddress){
- this.agentClass = agentClass;
- this.hpcAddress = hpcAddress;
- this.workspaceDirectory = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), agentClass);
- this.workspaceParentPath = Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName();
- }
-
- /**
- * Sets up a Slurm job by creating the job folder and the following files
- * under this folder:
- * - the input file.
- * - the Slurm script file.
- * - the JSON input file, which comes from the user request.
- *
- * @param jsonInput
- * @param slurmScript
- * @param input
- * @param timeStamp
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- public String setUpJob(String jsonInput, File slurmScript, File input, long timeStamp) throws IOException, SlurmJobException{
- String message = setUpJobOnAgentMachine(jsonInput, slurmScript, input, timeStamp);
- return message;
- }
-
- /**
- * Sets up a Slurm job by creating the job folder and the following files
- * under this folder:
- * - the input file.
- * - the Slurm script file.
- * - the JSON input file, which comes from the user request.
- *
- * @param jsonInput
- * @param slurmScript
- * @param input
- * @param executable
- * @param timeStamp
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- public String setUpJob(String jsonInput, File slurmScript, File input, File executable, long timeStamp) throws IOException, SlurmJobException{
- String message = setUpJobOnAgentMachine(jsonInput, slurmScript, input, executable, timeStamp);
- JSONObject obj = new JSONObject();
- obj.put("message", message);
- return obj.toString();
- }
-
-
- /**
- * Sets up the Slurm job for the current input.
- *
- * @param jsonString
- * @param slurmScript
- * @param input
- * @param timeStamp
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- private String setUpJobOnAgentMachine(String jsonString, File slurmScript, File input, long timeStamp) throws IOException, SlurmJobException{
- Workspace workspace = new Workspace();
- if(workspaceDirectory == null){
- LOGGER.error("Workspace directory has not been set!");
- return Status.JOB_SETUP_ERROR.getName();
- }else{
- return setUpSlurmJob(workspace, workspaceDirectory, jsonString, slurmScript, input, timeStamp);
- }
- }
-
- /**
- * Sets up the Slurm job for the current input.
- *
- * @param jsonString
- * @param slurmScript
- * @param input
- * @param executable
- * @param timeStamp
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- private String setUpJobOnAgentMachine(String jsonString, File slurmScript, File input, File executable, long timeStamp) throws IOException, SlurmJobException{
- Workspace workspace = new Workspace();
- if(workspaceDirectory == null){
- LOGGER.error("Workspace directory has not been set!");
- return Status.JOB_SETUP_ERROR.getName();
- }else{
- return setUpSlurmJob(workspace, workspaceDirectory, jsonString, slurmScript, input, executable, timeStamp);
- }
- }
-
- /**
- * Sets up the Slurm job for the current request.
- *
- * @param ws
- * @param workspaceFolder
- * @param jsonString
- * @param slurmScript
- * @param input
- * @param timeStamp
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- private String setUpSlurmJob(Workspace ws, File workspaceFolder, String jsonString, File slurmScript, File input, long timeStamp) throws IOException, SlurmJobException{
- File jobFolder = ws.createJobFolder(workspaceFolder.getAbsolutePath(), getHpcAddress(), timeStamp);
- LOGGER.info("Set up slurm job folder at: " + jobFolder);
-
- if(createAllFileInJobFolder(ws, workspaceFolder, jobFolder, jsonString, slurmScript, input) == null) {
- LOGGER.error("Could not create files within the slurm job folder.");
- return null;
- }
- return jobFolder.getName();
- }
-
- /**
- * Sets up the Slurm job for the current request.
- *
- * @param ws
- * @param workspaceFolder
- * @param jsonString
- * @param slurmScript
- * @param input
- * @param executable
- * @param timeStamp
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- private String setUpSlurmJob(Workspace ws, File workspaceFolder, String jsonString, File slurmScript, File input, File executable, long timeStamp) throws IOException, SlurmJobException{
- File jobFolder = ws.createJobFolder(workspaceFolder.getAbsolutePath(), getHpcAddress(), timeStamp);
- LOGGER.info("Set up slurm job folder at: " + jobFolder);
-
- if(createAllFileInJobFolder(ws, workspaceFolder, jobFolder, jsonString, slurmScript, input, executable) == null) {
- LOGGER.error("Could not create files within the slurm job folder.");
- return null;
- }
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }
-
- /**
- * Creates all files relevant for the current job, in particular, it
- * creates the following files:
- * - the input file in com (.com) format for running the job on an HPC
- * - the status file in text (.txt) format
- * - the input file in json (.json) format
- * - the Slurm script file in shell script (.sh) format
- *
- * @param ws
- * @param workspaceFolder
- * @param jobFolder
- * @param jsonString
- * @param slurmScript
- * @param input
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- private String createAllFileInJobFolder(Workspace ws, File workspaceFolder, File jobFolder, String jsonString, File slurmScript, File input) throws IOException, SlurmJobException{
- String inputFileMsg = ws.createInputFile(ws.getInputFilePath(jobFolder, getHpcAddress(), ws.getInputFileExtension(input)), input);
- if(inputFileMsg == null){
- return Status.JOB_SETUP_INPUT_FILE_ERROR.getName();
- }
- String statusFileMsg = ws.createStatusFile(workspaceFolder, ws.getStatusFilePath(jobFolder), getHpcAddress());
- if(statusFileMsg == null){
- return null;
- }
- String jsonInputFileMsg = ws.createJSONInputFile(workspaceFolder, ws.getJSONInputFilePath(jobFolder), jsonString);
- if(jsonInputFileMsg == null){
- return null;
- }
- String scriptFileMsg = ws.copyFile(slurmScript.getAbsolutePath(), jobFolder.getAbsolutePath().concat(File.separator).concat(slurmScript.getName()));
- if(scriptFileMsg == null){
- return null;
- }
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }
-
- /**
- * Creates all files relevant for the current job, in particular, it
- * creates the following files:
- * - the input file in zip (.zip) format for running the job on an HPC
- * - the status file in text (.txt) format
- * - the input file in json (.json) format
- * - the Slurm script file in shell script (.sh) format
- * - the executable file in java is in jar (.jar) format
- *
- * @param ws
- * @param workspaceFolder
- * @param jobFolder
- * @param jsonString
- * @param slurmScript
- * @param input
- * @param executable
- * @return
- * @throws IOException
- * @throws DFTAgentException
- */
- private String createAllFileInJobFolder(Workspace ws, File workspaceFolder, File jobFolder, String jsonString, File slurmScript, File input, File executable) throws IOException, SlurmJobException{
- String inputFileMsg = ws.createInputFile(ws.getInputFilePath(jobFolder, getHpcAddress(), ws.getInputFileExtension(input)), input);
- if(inputFileMsg == null){
- return Status.JOB_SETUP_INPUT_FILE_ERROR.getName();
- }
- String statusFileMsg = ws.createStatusFile(workspaceFolder, ws.getStatusFilePath(jobFolder), getHpcAddress());
- if(statusFileMsg == null){
- return null;
- }
- String jsonInputFileMsg = ws.createJSONInputFile(workspaceFolder, ws.getJSONInputFilePath(jobFolder), jsonString);
- if(jsonInputFileMsg == null){
- return null;
- }
- String scriptFileMsg = ws.copyFile(slurmScript.getAbsolutePath(), jobFolder.getAbsolutePath().concat(File.separator).concat(slurmScript.getName()));
- if(scriptFileMsg == null){
- return null;
- }
- String executableFileMsg = ws.copyFile(executable.getAbsolutePath(), jobFolder.getAbsolutePath().concat(File.separator).concat(executable.getName()));
- if(executableFileMsg == null){
- return null;
- }
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }
-
- /**
- * Shows the following statistics of Slurm jobs.
- * - Total number of jobs submitted
- * - Total number of jobs currently running
- * - Total number of jobs successfully completed
- * - Total number of jobs terminated with an error
- * - Total number of jobs not started yet
- *
- * @param input the JSON string specifying the return data format, e.g. JSON.
- * @return the statistics in JSON format if requested.
- */
- public String produceStatistics(String input) throws IOException, SlurmJobException{
- System.out.println("Received a request to send statistics.\n");
- LOGGER.info("Received a request to send statistics.\n");
- return "";
- }
-
- /**
- * Shows the following statistics of Slurm jobs.
- * - Total number of jobs submitted
- * - Total number of jobs currently running
- * - Total number of jobs successfully completed
- * - Total number of jobs terminated with an error
- * - Total number of jobs not started yet
- *
- * @return the statistics in HTML format.
- */
- public String showStatistics() throws IOException, SlurmJobException{
- System.out.println("Received a request to show statistics.\n");
- LOGGER.info("Received a request to show statistics.\n");
- return "";
- }
-
- /**
- * Produces the statistics about Slurm jobs.
- *
- * @param input in json format containing the format in which the result
- * should be codified, e.g. json. An example input file will look like
- * as follows:{"format":"json"}
- * @return a JSONObject containing statistics in the user requested format.
- * @throws IOException
- */
- public JSONObject getStatistics(String input) throws IOException{
- JSONObject obj = new JSONObject(input);
- String formatAccepts = obj.optString("format"); // json
- if(formatAccepts!=null && formatAccepts.toLowerCase().contains("json")){
- JobStatistics jobStatistics = new JobStatistics(workspaceDirectory);
- obj = new JSONObject();
- obj.put("Number of jobs currently running", jobStatistics.getJobsRunning());
- obj.put("Number of jobs successfully completed", jobStatistics.getJobsCompleted());
- obj.put("Number of jobs completing", jobStatistics.getJobsCompleting());
- obj.put("Number of jobs failed", jobStatistics.getJobsFailed());
- obj.put("Number of jobs pending", jobStatistics.getJobsPending());
- obj.put("Number of jobs preempted", jobStatistics.getJobsPreempted());
- obj.put("Number of jobs suspended", jobStatistics.getJobsSuspended());
- obj.put("Number of jobs stopped", jobStatistics.getJobsStopped());
- obj.put("Number of jobs terminated with an error", jobStatistics.getJobsErrorTerminated());
- obj.put("Number of jobs to start", jobStatistics.getJobsNotStarted());
- obj.put("Total number of jobs submitted", jobStatistics.getJobsSubmitted());
- }
- return obj;
- }
-
- /**
- * Produces the statistics about Slurm jobs.
- *
- * @return
- * @throws IOException
- */
- public String getStatistics() throws IOException{
- JobStatistics jobStatistics = new JobStatistics(workspaceDirectory);
- String statistics = jobStatistics.getHTMLHeader();
- statistics = statistics + "";
- statistics = statistics.concat(jobStatistics.getBodydivStart());
- statistics = statistics + "
";
- String headerText = "Statistics about jobs submitted to this agent are shown in the table below.
";
- statistics = statistics.concat(jobStatistics.getStatisticsTableHeader(headerText, "Property", "Value", "50%"));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs currently running", jobStatistics.getJobsRunning()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs successfully completed", jobStatistics.getJobsCompleted()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs completing", jobStatistics.getJobsCompleting()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs failed", jobStatistics.getJobsFailed()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs pending", jobStatistics.getJobsPending()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs preempted", jobStatistics.getJobsPreempted()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs suspended", jobStatistics.getJobsSuspended()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs stopped", jobStatistics.getJobsStopped()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs terminated with an error", jobStatistics.getJobsErrorTerminated()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs to start", jobStatistics.getJobsNotStarted()+""));
- statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Total number of jobs submitted", jobStatistics.getJobsSubmitted()+""));
- statistics = statistics + "
";
- statistics = statistics + "";
- statistics = statistics + "";
- statistics = statistics + "";
- return statistics;
- }
-
- /**
- * Monitors the currently running Slurm jobs to allow new jobs to start.
- * In doing so, it checks if the number of running jobs is less than the
- * maximum number of jobs allowed to run at a time.
- *
- */
- public void monitorJobs() throws SlurmJobException{
- try {
- if(!hostAvailabilityCheck(getHpcAddress(), 22)){
- System.out.println("The agent cannot connect to the HPC server with address " + getHpcAddress());
- session = null;
- return;
- }
- scheduledIteration++;
- if (session == null || scheduledIteration%10==0) {
- if(session!=null && session.isConnected()){
- session.disconnect();
- }
- System.out.println("Initialising a session.");
- session = jsch.getSession(slurmJobProperty.getHpcServerLoginUserName(), getHpcAddress(), 22);
- String pwd = slurmJobProperty.getHpcServerLoginUserPassword();
- session.setPassword(pwd);
- String privateKeyFilename =slurmJobProperty.getHpcServerPrivateKey();
-
- // Note that session.setConfig("PreferredAuthentications", ...) was removed because it will cause issues
- // when this code is executed in a container for unknown reasons
- try {
- if(!privateKeyFilename.isEmpty()){
- jsch.addIdentity(privateKeyFilename);
- }else{
- // Attempt to connect to a running instance of Pageant
- Connector con = new PageantConnector();
- IdentityRepository irepo = new RemoteIdentityRepository(con);
- jsch.setIdentityRepository(irepo);
- }
- } catch (AgentProxyException e) {
- LOGGER.info("Failed to detect Pageant, will authenticate using password");
- }
-
- session.setConfig("StrictHostKeyChecking", "no");
- session.connect();
- scheduledIteration = 0;
- }
- if(workspaceDirectory.isDirectory()){
- File[] jobFolders = workspaceDirectory.listFiles();
- updateRunningJobSet(jobFolders, jobsRunning);
-
- for(File jobFolder: jobFolders){
- if(!Utils.isJobCompleted(jobFolder, slurmJobProperty)){
- if(Utils.isJobRunning(jobFolder)){
- if(updateRunningJobsStatus(jobFolder)){
- if(jobsRunning.contains(jobFolder.getName())){
- jobsRunning.remove(jobFolder.getName());
- }
- }
- } else if(Utils.isJobNotStarted(jobFolder) && !jobsRunning.contains(jobFolder.getName())){
-
- if(jobsRunning.size() jobsRunning) throws IOException{
- for(File jobFolder: jobFolders){
- if(!(new File(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()))).exists()){
- LOGGER.info("SlurmJobAPI: job status file is not found, so the job folder with ID "+ jobFolder.getName()+" is being moved to the failed job folder.");
- System.out.println("SlurmJobAPI: job status file is not found, so the job folder with ID "+ jobFolder.getName()+" is being moved to the failed job folder.");
- try{
- Utils.moveToFailedJobsFolder(jobFolder, slurmJobProperty);
- }catch(Exception e){
- LOGGER.info("SlurmJobAPI: failed to move the job folder with ID "+jobFolder.getName()+" to the failed job folder.");
- System.out.println("SlurmJobAPI: failed to move the job folder with ID "+jobFolder.getName()+" to the failed job folder.");
- }
- continue;
- }
- try {
- if (Utils.isJobRunning(jobFolder)) {
- jobsRunning.add(jobFolder.getName());
- }
- } catch (Exception e) {
- LOGGER.info("SlurmJobAPI: failed to check the status of the job with ID "+jobFolder.getName()+ " while checking if it was running.");
- System.out.println("SlurmJobAPI: failed to check the status of the job with ID "+jobFolder.getName()+ " while checking if it was running.");
- }
- }
- }
-
- /**
- * Starts running Slurm jobs which were set up before.
- *
- * @param jobFolder
- * @throws SftpException
- * @throws JSchException
- * @throws IOException
- * @throws UnknownHostException
- * @throws InterruptedException
- */
- private boolean runNotStartedJob(File jobFolder) throws SftpException, JSchException, IOException, UnknownHostException, InterruptedException{
- // A counter variable to count the number of mandatory files.
- int countNumberOfFilesSetInProperties = 0;
-
- // Checks if the script file (including name and extension) for the current Slurm job is provided.
- if(slurmJobProperty.getSlurmScriptFileName()==null || slurmJobProperty.getSlurmScriptFileName().isEmpty()){
- throw new IOException("SlurmJobAPI: Slurm script file name and extension are not provided.");
- }else{
- countNumberOfFilesSetInProperties++;
- }
- // Checks if the input file name for the current Slurm job is provided.
- if(slurmJobProperty.getInputFileName()==null || slurmJobProperty.getInputFileName().isEmpty()){
- throw new IOException("SlurmJobAPI: input file name is not provided.");
- }else{
- countNumberOfFilesSetInProperties++;
- }
- // Checks if input file extension for the current Slurm job is provided.
- if(slurmJobProperty.getInputFileExtension()==null || slurmJobProperty.getInputFileExtension().isEmpty()){
- throw new IOException("SlurmJobAPI: input file extension is not provided.");
- }
- // Checks if the JSON input file name for the current Slurm job is provided.
- if(slurmJobProperty.getJsonInputFileName()==null || slurmJobProperty.getJsonInputFileName().isEmpty()){
- throw new IOException("SlurmJobAPI: JSON input file name is not provided.");
- }else{
- countNumberOfFilesSetInProperties++;
- }
- // Checks if the JSON input file extension for the current Slurm job is provided.
- if(slurmJobProperty.getJsonFileExtension()==null || slurmJobProperty.getJsonFileExtension().isEmpty()){
- throw new IOException("SlurmJobAPI: JSON file extension is not provided.");
- }
- // Checks if the executable file (including name and extension) for the current Slurm job is provided.
- if(slurmJobProperty.getExecutableFile()!=null && !slurmJobProperty.getExecutableFile().isEmpty()){
- countNumberOfFilesSetInProperties++;
- }
- int countNumberOfFilesInJobFolder = 0;
- // Checks the availability of the following four mandatory files in the current job folder:
- // 1. Slurm script file (e.g., Slurm.sh)
- // 2. Input file (e.g., input.zip or input.com)
- // 3. JSON input file (e.g., input.json)
- // 4. Status file (e.g., status.txt)
- for(File file:jobFolder.listFiles()){
- if(file.getName().equalsIgnoreCase(slurmJobProperty.getSlurmScriptFileName())){
- countNumberOfFilesInJobFolder++;
- }
- if(file.getName().endsWith(slurmJobProperty.getInputFileExtension())){
- countNumberOfFilesInJobFolder++;
- }
- if(file.getName().equalsIgnoreCase(slurmJobProperty.getJsonInputFileName().concat(slurmJobProperty.getJsonFileExtension()))){
- countNumberOfFilesInJobFolder++;
- }
- if(file.getName().equalsIgnoreCase(Status.STATUS_FILE.getName())){
- countNumberOfFilesInJobFolder++;
- }
- if(slurmJobProperty.getExecutableFile()!=null && file.getName().equalsIgnoreCase(slurmJobProperty.getExecutableFile())){
- countNumberOfFilesInJobFolder++;
- }
- }
- try{
- // If all files set through properties are not available in a job folder, it
- // deletes the folder.
- System.out.println("countNumberOfFilesInJobFolder:"+countNumberOfFilesInJobFolder);
- System.out.println("countNumberOfFilesSetInProperties:"+countNumberOfFilesSetInProperties);
- if(!(countNumberOfFilesSetInProperties>=3 && countNumberOfFilesSetInProperties+1==countNumberOfFilesInJobFolder)){
- LOGGER.info("SlurmJobAPI: all mandatory files are not found, so the job folder with ID "+ jobFolder.getName()+" is deleted.");
- System.out.println("SlurmJobAPI: all mandatory files are not found, so the job folder with ID "+ jobFolder.getName()+" is deleted.");
- Utils.moveToFailedJobsFolder(jobFolder, slurmJobProperty);
- return false;
- }
- }catch(Exception e){
- LOGGER.info("SlurmJobAPI: all mandatory files are not found and an attempt to move the job folder with ID "
- +jobFolder.getName()+" to the failed job folder is not successful.");
- System.out.println("SlurmJobAPI: all mandatory files are not found and an attempt to move the job folder with ID "
- +jobFolder.getName()+" to the failed job folder is not successful.");
- }
-
- try{
- startJob(jobFolder.getName(), Arrays.asList(jobFolder.listFiles()));
- }catch(Exception e){
- LOGGER.info("SlurmJobAPI: the Slurm Job with ID "+jobFolder.getName()+" could not be started.", e);
- System.out.println("SlurmJobAPI: the Slurm Job with ID "+jobFolder.getName()+" could not be started.");
- return false;
- }
- return true;
- }
-
- /**
- * Starts a Slurm job.
- *
- * @param job
- * @param jobFiles
- * @throws SftpException
- * @throws JSchException
- * @throws IOException
- * @throws UnknownHostException
- * @throws InterruptedException
- */
- private void startJob(String job, List jobFiles) throws SftpException, JSchException, IOException, UnknownHostException, InterruptedException{
- // On HPC, the job folder and files have the address of machine where DFT Agent runs.
- // Therefore, the HPC address is replaced with the address of Agent machine.
- job = job.replace(getHpcAddress(), Utils.getMachineAddress());
- String jobFolderOnHPC = createJobFolder(job);
- if(jobFolderOnHPC!=null){
- uploadFiles(jobFolderOnHPC, jobFiles);
- }
- }
-
- /**
- * For running the current job, it uploads both the input file and
- * Slurm script to an HPC.
- *
- * @param jobFolderOnHPC
- * @param jobFiles
- * @throws SftpException
- * @throws JSchException
- * @throws IOException
- * @throws UnknownHostException
- * @throws InterruptedException
- */
- private void uploadFiles(String jobFolderOnHPC, List jobFiles) throws SftpException, JSchException, IOException, UnknownHostException, InterruptedException{
- String replacedInputFileName = "";
- String statusFileAbsolutePath = "";
- String jobId = "";
-
- for(File jobFile:jobFiles){
-
- if(jobFile.getAbsolutePath().endsWith(slurmJobProperty.getInputFileExtension())){
- replacedInputFileName = getInputFileNameReplaced(jobFile.getAbsolutePath());
- String inputFileNameOnHPC = jobFolderOnHPC.concat("/").concat(replacedInputFileName);
- uploadFile(jobFile.getAbsolutePath(), inputFileNameOnHPC);
- replaceFileContent(jobFolderOnHPC, inputFileNameOnHPC);
-
- }else if(!jobFile.getAbsolutePath().endsWith(Property.STATUS_FILE_NAME.getPropertyName())){
- if(jobFile.getAbsolutePath().endsWith(".sh")){
- Utils.translateLineEndingIntoUnix(new File(jobFile.getAbsolutePath()));
- }
- uploadFile(jobFile.getAbsolutePath(), jobFolderOnHPC);
- }
-
- if(jobFile.getAbsolutePath().endsWith(Property.STATUS_FILE_NAME.getPropertyName())){
- statusFileAbsolutePath = jobFile.getAbsolutePath();
- }
- }
-
- LOGGER.info("Attemping to run job folder on HPC at: " + jobFolderOnHPC);
- jobId = runSlurmJob(jobFolderOnHPC, replacedInputFileName);
- if(!jobId.isEmpty()){
- Utils.addJobId(statusFileAbsolutePath, jobId);
- }
- }
-
- /**
- * Produces the command to go to the current job directory and to run
- * the job.
- *
- * @param jobFolderOnHPC
- * @param inputFile
- * @return
- * @throws JSchException
- * @throws IOException
- */
- private String runSlurmJob(String jobFolderOnHPC, String inputFile) throws JSchException, IOException{
- inputFile = inputFile.replace(slurmJobProperty.getInputFileExtension(), "");
- String command = "cd "
- .concat(jobFolderOnHPC)
- .concat(" && ")
- .concat("sbatch --job-name=").concat(inputFile)
- .concat(" ")
- .concat(Property.SLURM_SCRIPT_FILE_NAME.getPropertyName());
-
- LOGGER.info("Using command: " + command);
- return runSlurmJob(command);
- }
-
- /**
- * Runs a Slurm job.
- *
- * @param command
- * @return
- * @throws JSchException
- * @throws IOException
- */
- private String runSlurmJob(String command) throws JSchException, IOException{
- ArrayList outputs = executeCommand(command);
- if (outputs == null) {
- return null;
- }
- String jobId = getJobId(outputs);
- System.out.println("Job id:" + jobId);
- return jobId;
- }
-
- /**
- * Modifies the names of the check point file and log file provided in
- * the input file after copying this into the HPC cluster.
- *
- * @param jobFolderOnHPC
- * @param replacedJobFileName
- * @throws JSchException
- * @throws IOException
- * @throws UnknownHostException
- */
- private void replaceFileContent(String jobFolderOnHPC, String replacedJobFileName) throws JSchException, IOException, UnknownHostException{
- String command = "cd ".concat(jobFolderOnHPC).concat(" && ")
- .concat("sed -i 's/").concat(getHpcAddress())
- .concat("/").concat(Utils.getMachineAddress()).concat("/g' ").concat(replacedJobFileName);
- executeCommand(command);
- }
-
- /**
- * Modifies the name of the input file while copying this into the HPC cluster.
- *
- * @param inputFile
- * @return
- * @throws UnknownHostException
- */
- private String getInputFileNameReplaced(String inputFile) throws UnknownHostException{
- String tokens[];
- if(inputFile.contains("/")){
- tokens = inputFile.split("/");
- }else{
- tokens = inputFile.split("\\\\");
- }
- return tokens[tokens.length-1].replace(getHpcAddress(), Utils.getMachineAddress());
- }
-
- private String createJobFolder(String job) throws JSchException, IOException{
- // Creates the "mkdir" (make directory) command to create the workspace/jobspace directory.
- String command = "mkdir /home/".concat(slurmJobProperty.getHpcServerLoginUserName()).concat("/").concat(workspaceDirectory.getName());
- // Executes the command to create the workspace/jobspace directory.
- executeCommand(command);
- // Creates the command to create the job directory.
- command = "mkdir /home/".concat(slurmJobProperty.getHpcServerLoginUserName()).concat("/").concat(workspaceDirectory.getName()).concat("/").concat(job);
- // Executes the command for creating the job directory.
- executeCommand(command);
- return "/home/".concat(slurmJobProperty.getHpcServerLoginUserName()).concat("/").concat(workspaceDirectory.getName()).concat("/").concat(job);
- }
-
- /**
- * Updates the status of a currently running job.
- *
- * @param jobFolder
- * @return
- * @throws JSchException
- * @throws SftpException
- * @throws IOException
- * @throws InterruptedException
- */
- private boolean updateRunningJobsStatus(File jobFolder)
- throws JSchException, SftpException, IOException, InterruptedException {
- boolean status = false;
- try{
- File statusFile = Utils.getStatusFile(jobFolder);
- status = updateRunningJobsStatus(jobFolder.getName(), statusFile);
- }catch(Exception e){
- LOGGER.info("SlurmJobAPI: failed to update the status of the job with ID "+jobFolder.getName()+" while checking if it was still running.");
- }
- return status;
- }
-
- /**
- * Updates the latest status of the running jobs.
- *
- * @param runningJob
- * @param statusFile
- * @return
- * @throws JSchException
- * @throws SftpException
- * @throws IOException
- * @throws InterruptedException
- */
- private boolean updateRunningJobsStatus(String runningJob, File statusFile) throws JSchException, SftpException, IOException, InterruptedException{
- if(statusFile!=null){
- if(!isJobRunning(statusFile)){
- if(slurmJobProperty.getOutputFileExtension().trim().toLowerCase().equals(".log")){
- if(outputFileExist(Utils.getLogFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress()))){
- try{
- downloadFile(Utils.getLogFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress()), Utils.getJobOutputFilePathOnAgentPC(runningJob, workspaceDirectory, runningJob, Status.EXTENSION_LOG_FILE.getName()));
- updateStatusForErrorTermination(statusFile, Utils.getJobOutputFilePathOnAgentPC(runningJob, workspaceDirectory, runningJob, Status.EXTENSION_LOG_FILE.getName()));
- }catch(Exception e){
- Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName());
- }
- }else{
- Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName());
- }
- }else{
- if(outputFileExist(Utils.getOutputFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress(), slurmJobProperty.getOutputFileName().concat(slurmJobProperty.getOutputFileExtension())))){
- try{
- downloadFile(Utils.getOutputFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress(), slurmJobProperty.getOutputFileName().concat(slurmJobProperty.getOutputFileExtension())), Utils.getJobOutputFilePathOnAgentPC(runningJob, workspaceDirectory, slurmJobProperty.getOutputFileName(), slurmJobProperty.getOutputFileExtension()));
- }catch(Exception e){
- Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName());
- }
- }else{
- Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName());
- }
- }
- deleteJobOnHPC(Utils.getJobFolderPathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress()));
- return true;
- }
- }
- return false;
- }
-
- /**
- * Checks if the output file exists. If it does, it returns true, it returns false otherwise.
- *
- * @param fileAbsoultePath
- * @return
- * @throws JSchException
- * @throws IOException
- */
- private boolean outputFileExist(String fileAbsoultePath) throws JSchException, IOException{
- String command = "[ -f "+fileAbsoultePath+" ] && echo "+Status.JOB_OUTPUT_FILE_EXIST_MESSAGE.getName();
- ArrayList outputs = executeCommand(command);
- if(outputs!=null && outputs.size()>0){
- if(outputs.contains(Status.JOB_OUTPUT_FILE_EXIST_MESSAGE.getName())){
- return true;
- }
- }
- return false;
- }
-
- /**
- * Extracts from the log file if the current job terminated normally or
- * with error. In the case of error termination, updates the status of
- * the log file accordingly.
- *
- * @param statusFile
- * @param jobFolderPathOnAgentPC
- */
- private void updateStatusForErrorTermination(File statusFile, String jobFolderPathOnAgentPC) throws IOException{
- if(Utils.isErrorTermination(jobFolderPathOnAgentPC)){
- Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName());
- }
- }
-
- /**
- * Checks if a job is running using the job id.
- *
- * @param statusFile
- * @return
- * @throws JSchException
- * @throws IOException
- * @throws InterruptedException
- */
- private boolean isJobRunning(File statusFile) throws JSchException, IOException, InterruptedException{
- String jobId = Utils.getJobId(statusFile.getAbsolutePath());
- if(jobId==null){
- return false;
- } else{
- boolean isJobRunning = isJobRunning(jobId , statusFile);
- if(isJobRunning){
- return true;
- }else{
- Utils.modifyStatus(statusFile.getAbsolutePath(), Status.STATUS_JOB_COMPLETED.getName());
- return false;
- }
- }
- }
-
- /**
- * Checks the status of job on the HPC where it was submitted.
- *
- * @param jobId
- * @param statusFile
- * @return
- * @throws IOException
- * @throws InterruptedException
- */
- private boolean isJobRunning(String jobId, File statusFile) throws JSchException, IOException, InterruptedException{
- String command = "squeue -j " + jobId;
- ArrayList outputs = executeCommand(command);
- boolean jobRunning = isJobRunning(outputs, statusFile);
- return jobRunning;
- }
-
- /**
- * Analyses the outputs following the execution of the
- * job status check command to understand if the job
- * is still running or terminated.
- *
- * @param outputs
- * @return
- */
- private boolean isJobRunning(ArrayList outputs, File statusFile) throws IOException{
- if(outputs!=null && outputs.size()<=1){
- return false;
- }
- if(outputs!=null){
- String[] headers = outputs.get(0).split("\\s+");
- String[] values = outputs.get(1).split("\\s+");
- for(int i=0; ii){
-// updateStatus(values[i], statusFile);
- }
- }
- }
- }
- return true;
- }
-
- /**
- * Extracts the job id from the
- *
- * @param outputs
- * @return
- */
- private String getJobId(ArrayList outputs){
- for(String output: outputs){
- if(output.contains("Submitted batch job")){
- String tokens[] = output.split(" ");
- if(tokens.length>=4){
- return tokens[3].trim();
- }
- }
- }
- return null;
- }
-
- /**
- * Deletes the job folder including all files and folders belonging to it.
- *
- * @param jobFolder the absolute path to the job folder.
- * @throws JSchException
- * @throws IOException
- * @throws InterruptedException
- */
- private void deleteJobOnHPC(String jobFolder) throws JSchException, IOException, InterruptedException{
- String command = "rm -r " + jobFolder;
- executeCommand(command);
- }
-
- /**
- * Reads the output of the most recently executed command.
- *
- * @param br
- * @return
- * @throws IOException
- */
- public ArrayList readCommandOutput(BufferedReader br) throws IOException{
- if(br == null)
- throw new IOException("The reader is not initialised.");
- String line;
- ArrayList outputs = new ArrayList<>();
- while((line=br.readLine())!=null){
- outputs.add(line);
- }
- br.close();
- return outputs;
- }
-
- /**
- * Displays every item in a list of string array.
- *
- * @param list
- */
- public void displayArray(ArrayList list){
- for(String item:list){
- System.out.println(item);
- }
- }
-
- /**
- * Uploads a file from the source folder belonging to any machine to
- * the destination folder on the same machine or any other machine.
- *
- * @param src
- * @param dest
- * @throws JSchException
- * @throws SftpException
- */
- public void uploadFile(String src, String dest) throws JSchException, SftpException{
- System.out.println("Establishing a channel to transfer "+src+" to "+dest);
- ChannelSftp sftpChannel = (ChannelSftp) session.openChannel("sftp");
- sftpChannel.connect();
-
- sftpChannel.put(src, dest);
- sftpChannel.disconnect();
- System.out.println("Closing the channel.");
- }
-
- /**
- * Downloads the log file.
- *
- * @param src the file to be downloaded
- * @param dest the path under which the source file will be downloaded
- * or the absolute path containing the path and the file name.
- * @throws JSchException
- * @throws SftpException
- */
- public void downloadFile(String src, String dest) throws JSchException, SftpException{
- System.out.println("Establishing a channel to transfer "+src+" to "+dest);
- ChannelSftp sftpChannel = (ChannelSftp) session.openChannel("sftp");
- sftpChannel.connect();
- sftpChannel.get(src, dest);
- sftpChannel.disconnect();
- System.out.println("Closing the channel.");
- }
-
- /**
- * Deletes a folder or file name from an HPC, if the complete path is provided.
- * For example, to delete a folder called "test", user needs to provide "rds/.../test"
- *
- * @param folderOrFileName
- * @throws JSchException
- * @throws SftpException
- */
- public void deleteFolderOrFile(String folderOrFileName) throws JSchException, SftpException, IOException{
- executeCommand("rm -r "+folderOrFileName);
- }
-
- /**
- * Executes any command that can be run on the Linux console.
- *
- * @param Command
- * @return
- * @throws JSchException
- * @throws IOException
- */
- public ArrayList executeCommand(String Command) throws JSchException, IOException{
- ArrayList outputs = null;
- System.out.println("Establishing a channel to perform the following command:" + Command);
- Channel execChannel = session.openChannel("exec");
- ((ChannelExec) execChannel).setCommand(Command);
- execChannel.setInputStream(null);
- ((ChannelExec) execChannel).setErrStream(System.err);
- BufferedReader stdInput = new BufferedReader(new InputStreamReader(execChannel.getInputStream()));
- execChannel.connect();
- outputs = readCommandOutput(stdInput);
- execChannel.disconnect();
- System.out.println("Closing the channel.");
- return outputs;
- }
-
- /**
- * Indicates if a server is online.
- *
- * @param server refers to the server address
- * @param port refers to the port number
- * @return
- */
- public boolean hostAvailabilityCheck(String server, int port) throws IOException {
- boolean available = true;
-// try (final Socket dummy = new Socket(server, port)){
-// } catch (UnknownHostException | IllegalArgumentException e) {
-// available = false;
-// }
- return available;
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessing.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessing.java
deleted file mode 100644
index 1f97fa416..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessing.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.SftpException;
-
-import uk.ac.cam.cares.jps.base.util.FileUtil;
-
-
-public class PostProcessing {
- /**
- * Updates the output status of a completed job.
- *
- * @param jobFolder
- * @return
- * @throws JSchException
- * @throws SftpException
- * @throws IOException
- * @throws InterruptedException
- */
- public static boolean updateJobOutputStatus(File jobFolder)
- throws IOException {
- File statusFile = Utils.getStatusFile(jobFolder);
- return updateJobOutputStatus(statusFile, Status.OUTPUT_PROCESSED.getName());
- }
-
- /**
- * Updates the latest status of the running jobs.
- *
- * @param runningJob
- * @param statusFile
- * @return
- * @throws JSchException
- * @throws SftpException
- * @throws IOException
- * @throws InterruptedException
- */
- private static boolean updateJobOutputStatus(File statusFile, String status) throws IOException{
- if(statusFile!=null){
- modifyOutputStatus(statusFile.getAbsolutePath(), Status.OUTPUT_PROCESSED.getName());
- return true;
- }
- return false;
- }
-
- /**
- * Modifies the output to processed in the status file.
- *
- * @param filePath the path to the status file.
- * @param status can be empty ("") or "processed".
- * @throws IOException
- */
- public static void modifyOutputStatus(String filePath, String status) throws IOException{
- List fileContent = new ArrayList<>();
- BufferedReader br = FileUtil.openSourceFile(filePath);
- String line;
- while((line=br.readLine())!=null){
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_OUTPUT.getName())) {
- line = Status.ATTRIBUTE_JOB_OUTPUT.getName().concat(" ").concat(status);
- }
- fileContent.add(line);
- }
- br.close();
- BufferedWriter bw = FileUtil.openBufferedWriter(filePath);
- for(String lineContent:fileContent){
- bw.write(lineContent.concat("\n"));
- }
- bw.flush();
- bw.close();
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Property.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Property.java
deleted file mode 100644
index 1ee69f85c..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Property.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-/**
- * This enumerated list defines the name of important properties of
- * Slurm jobs. Some example properties are:
- * - the name of the agent class
- * - the name of the workspace folder on the machine where the agent runs
- * - the name of the workspace folder on HPC where DFT calculations run
- *
- * @author Feroz Farazi(msff2@cam.ac.uk)
- *
- */
-public enum Property {
-
- JOB_WORKSPACE_PARENT_DIR(System.getProperty("user.home")),
- CHK_POINT_FILE_EXTENSION(".chk"),
- STATUS_FILE_NAME("status.txt"),
- JSON_INPUT_FILE_NAME("input.json"),
- SLURM_SCRIPT_FILE_NAME("Slurm.sh");
-
- private String propertyName;
- private int value;
- private Property(String propertyName){
- this.propertyName = propertyName;
- }
-
- public String getPropertyName(){
- return propertyName;
- }
-
- private Property(final int newValue){
- value = newValue;
- }
-
- public int getValue(){
- return value;
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJob.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJob.java
deleted file mode 100644
index 75d29c2f0..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJob.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-public class SlurmJob {
- private String jobFolderName;
-
- public String getJobFolderName() {
- return jobFolderName;
- }
-
- public void setJobFolderName(String jobFolderName) {
- this.jobFolderName = jobFolderName;
- }
- /**
- * To create a new job folder name, this method is used.
- *
- * @param hpcAddress
- * @return
- */
- public String getNewJobFolderName(String hpcAddress){
- setJobFolderName(hpcAddress.concat("_").concat("" + Utils.getTimeStamp()));
- return getJobFolderName();
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobException.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobException.java
deleted file mode 100644
index 82ae1fc9e..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-/**
- * The exception class to catch all possible exceptions within the
- * Slurm Job API.
- *
- * @author msff2
- *
- */
-public class SlurmJobException extends Exception{
- public SlurmJobException() {
- super();
- }
-
- public SlurmJobException(String message) {
- super(message);
- }
-
- public SlurmJobException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public SlurmJobException(Throwable cause) {
- super(cause);
- }
-
- protected SlurmJobException(String message, Throwable cause,
- boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Status.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Status.java
deleted file mode 100644
index a06eaed83..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Status.java
+++ /dev/null
@@ -1,57 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-/**
- * Starting from the submission to the completion a Slurm job goes through
- * different statuses, e.g. pending, running and completed. This java class
- * contains all possible statuses and corresponding messages as constants.
- *
- * @author msff2
- *
- */
-public enum Status {
- STATUS_FILE("status.txt"),
- ATTRIBUTE_JOB_STATUS("JobStatus:"),
- STATUS_JOB_NOT_STARTED("not started"),
- STATUS_JOB_COMPLETED("completed"),
- STATUS_JOB_COMPLETING("completing"),
- STATUS_JOB_FAILED("failed"),
- STATUS_JOB_PENDING("pending"),
- STATUS_JOB_PREEMPTED("preempted"),
- STATUS_JOB_RUNNING("running"),
- STATUS_JOB_SUSPENDED("suspended"),
- STATUS_JOB_STOPPED("stopped"),
- STATUS_JOB_ERROR_TERMINATED("error termination"),
- ATTRIBUTE_JOB_ID("JobId:"),
- ATTRIBUTE_AGENT_ID("AgentId:"),
- ATTRIBUTE_HPC_ADDRESS("HPCAddress:"),
- ATTRIBUTE_JOB_OUTPUT("JobOutput:"),
- OUTPUT_PROCESSED("processed"),
- EXTENSION_SLURM_FILE(".sh"),
- EXTENSION_LOG_FILE(".log"),
- EXTENSION_JSON_FILE(".json"),
- JOB_SETUP_SUCCESS_MSG("The job request has been processed successfully."),
- JOB_SETUP_JSON_ERROR("The JSON request is ill formatted."),
- JOB_SETUP_SLURM_SCRIPT_ERROR("The Slurm script could not be downloaded."),
- JOB_SETUP_INPUT_FILE_ERROR("The input file for the job could not be created."),
- JOB_SETUP_ERROR("The requested job could not be set up, therefore, resubmit the job."),
- JOB_LOG_MSG_ERROR_TERMINATION("error termination"),
- JOB_LOG_MSG_NORMAL_TERMINATION("normal termination"),
- JOB_OUTPUT_FILE_EXIST_MESSAGE("The file exist."),
- JOB_SQUEUE_STATUS_COMPLETED("CD"),
- JOB_SQUEUE_STATUS_COMPLETING("CG"),
- JOB_SQUEUE_STATUS_FAILED("F"),
- JOB_SQUEUE_STATUS_PENDING("PD"),
- JOB_SQUEUE_STATUS_PREEMPTED("PR"),
- JOB_SQUEUE_STATUS_RUNNING("R"),
- JOB_SQUEUE_STATUS_SUSPENDED("S"),
- JOB_SQUEUE_STATUS_STOPPED("ST");
-
- private String property;
- private Status(String property){
- this.property = property;
- }
-
- public String getName(){
- return property;
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Utils.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Utils.java
deleted file mode 100644
index 02f83ecb6..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Utils.java
+++ /dev/null
@@ -1,626 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-
-import uk.ac.cam.cares.jps.base.slurm.job.configuration.SlurmJobProperty;
-
-public class Utils{
- private static Logger LOGGER = LogManager.getLogger(Utils.class);
- public static long previousTimeStamp;
-
- /**
- * Generate unique time stamp to be used in naming Slurm jobs.
- *
- * @return
- */
- public static long getTimeStamp(){
- long currentTimeStamp = System.nanoTime();
- while(!(currentTimeStamp > previousTimeStamp)){
- currentTimeStamp = System.nanoTime();
- }
- previousTimeStamp = currentTimeStamp;
- return currentTimeStamp;
- }
-
- /**
- * Creates an instance of the BufferedWriter class.
- *
- * @param filePathPlusName the path plus name of the file being written
- * @return
- * @throws IOException
- */
- public static BufferedWriter openBufferedWriter(String filePathPlusName) throws IOException{
- return new BufferedWriter(new FileWriter(filePathPlusName));
- }
-
- /**
- * Creates and returns an instance of the BufferedReader class.
- *
- * @param filePathPlusName
- * the path plus name of the file being read
- * @return
- * @throws IOException
- */
- public static BufferedReader openSourceFile(String filePathPlusName)
- throws IOException {
- return new BufferedReader(new InputStreamReader(new FileInputStream(
- filePathPlusName), "UTF-8"));
- }
-
- /**
- * Check if a job is completed.
- *
- * @param jobFolder
- * @return
- * @throws IOException
- */
- public static boolean isJobCompleted(File jobFolder) throws IOException{
- return isJobFinished(jobFolder, jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()));
- }
-
- /**
- * Check if a job is completed.
- *
- * @param jobFolder
- * @param slurmJobProperty
- * @return
- * @throws IOException
- */
- public static boolean isJobCompleted(File jobFolder, SlurmJobProperty slurmJobProperty) throws IOException{
- boolean status = false;
- try {
- status = isJobFinished(jobFolder, jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()), slurmJobProperty);
- } catch (Exception e) {
- LOGGER.info("SlurmJobAPI: failed to check the status of the job with ID " + jobFolder.getName());
- }
- return status;
- }
-
- /**
- * Check if a job is completed with erroneous or incomplete output.
- *
- * @param jobFolder
- * @return
- * @throws IOException
- */
- public static boolean isJobErroneouslyCompleted(File jobFolder) throws IOException{
- return isJobErroneouslyCompleted(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()));
- }
-
- /**
- * Checks if a job is still running.
- *
- * @param jobFolder
- * @return
- * @throws IOException
- */
- public static boolean isJobRunning(File jobFolder) throws IOException{
- boolean status = false;
- try{
- status = isJobRunning(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()));
- }catch(Exception e){
- LOGGER.info("SlurmJobAPI: failed to check the status of the job with ID "+jobFolder.getName());
- }
- return status;
- }
-
- /**
- * Checks if a job is not started yet.
- *
- * @param jobFolder
- * @return
- * @throws IOException
- */
- public static boolean isJobNotStarted(File jobFolder) throws IOException{
- return isJobNotStarted(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()));
- }
-
- /**
- * Check the status if a job finished.
- *
- * @param jobFolder the folder that contains the job.
- * @param statusFilePath the file that contains the status of the job.
- * @return
- * @throws IOException
- */
- public static boolean isJobFinished(File jobFolder, String statusFilePath) throws IOException{
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){
- if(line.contains(Status.STATUS_JOB_COMPLETED.getName())){
- statusFile.close();
- return true;
- }
- if(line.contains(Status.STATUS_JOB_ERROR_TERMINATED.getName())){
- statusFile.close();
- return true;
- }
- }
- }
- statusFile.close();
- return false;
- }
-
-
- /**
- * Check the status if a job finished.
- *
- * @param jobFolder the folder that contains the job.
- * @param statusFilePath the file that contains the status of the job.
- * @param slurmJobProperty the variable that allows to read different
- * properties including agentClass, completed jobs folder prefix, etc.
- *
- * @return
- * @throws IOException
- */
- public static boolean isJobFinished(File jobFolder, String statusFilePath, SlurmJobProperty slurmJobProperty) throws IOException{
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){
- if(line.contains(Status.STATUS_JOB_COMPLETED.getName())){
- statusFile.close();
- if(isJobPostProcessed(jobFolder, statusFilePath)){
- moveToCompletedJobsFolder(jobFolder, slurmJobProperty);
- }
- return true;
- }
- if(line.contains(Status.STATUS_JOB_ERROR_TERMINATED.getName())){
- statusFile.close();
- moveToFailedJobsFolder(jobFolder, slurmJobProperty);
- return true;
- }
- }
- }
- statusFile.close();
- return false;
- }
-
- /**
- * Check the status if a job post-processed.
- *
- * @param jobFolder the folder that contains the job.
- * @param statusFilePath the file that contains the status of the job.
- * @return
- * @throws IOException
- */
- public static boolean isJobPostProcessed(File jobFolder, String statusFilePath) throws IOException{
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().startsWith(Status.ATTRIBUTE_JOB_OUTPUT.getName())){
- if(line.contains(Status.OUTPUT_PROCESSED.getName())){
- statusFile.close();
- return true;
- }
- }
- }
- statusFile.close();
- return false;
- }
-
- /**
- * Check the status if a job produced erroneous or incomplete output.
- *
- * @param statusFilePath
- * @return
- * @throws IOException
- */
- public static boolean isJobErroneouslyCompleted(String statusFilePath) throws IOException{
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){
- if(line.contains(Status.STATUS_JOB_ERROR_TERMINATED.getName())){
- statusFile.close();
- return true;
- }
- }
- }
- statusFile.close();
- return false;
- }
-
- public static boolean isJobOutputProcessed(File jobFolder) throws IOException{
- return isJobOutputProcessed(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()));
- }
-
- /**
- * Check the status if a job is currently running.
- *
- * @param statusFilePath the absolute path to the status file.
- * @return
- * @throws IOException
- */
- public static boolean isJobOutputProcessed(String statusFilePath) throws IOException{
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().startsWith(Status.ATTRIBUTE_JOB_OUTPUT.getName())){
- if(line.contains(Status.OUTPUT_PROCESSED.getName())){
- statusFile.close();
- return true;
- }
- }
- }
- statusFile.close();
- return false;
- }
-
- /**
- * Check the status if a job is currently running.
- *
- * @param statusFilePath the absolute path to the status file.
- * @return
- * @throws IOException
- */
- public static boolean isJobRunning(String statusFilePath) throws IOException{
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){
- if(line.contains(Status.STATUS_JOB_RUNNING.getName())
- || line.contains(Status.STATUS_JOB_COMPLETING.getName())
- || line.contains(Status.STATUS_JOB_PENDING.getName())){
- statusFile.close();
- return true;
- }
- }
- }
- statusFile.close();
- return false;
- }
-
- protected static boolean isStatusFileOpen = false;
-
- /**
- * Check the status if a job is not started yet.
- *
- * @param statusFilePath the absolute path to the status file.
- * @return
- * @throws IOException
- */
- public static boolean isJobNotStarted(String statusFilePath) throws IOException{
- if(isStatusFileOpen){
- return false;
- }
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- isStatusFileOpen = true;
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){
- if(line.contains(Status.STATUS_JOB_NOT_STARTED.getName())){
- statusFile.close();
- isStatusFileOpen = false;
- return true;
- }
- }
- }
- statusFile.close();
- isStatusFileOpen = false;
- return false;
- }
-
- /**
- * Reads the job id from the status file.
- *
- * @param statusFilePath the absolute path to the status file.
- * @return
- * @throws IOException
- */
- public static String getJobId(String statusFilePath) throws IOException{
- BufferedReader statusFile = Utils.openSourceFile(statusFilePath);
- String line;
- while((line=statusFile.readLine())!=null){
- if(line.trim().toLowerCase().startsWith(Status.ATTRIBUTE_JOB_ID.getName().toLowerCase())){
- String tokens[] = line.trim().split(":");
- if(tokens.length>=2 && tokens[1].trim().length()>0){
- statusFile.close();
- return tokens[1].trim();
- }
- }
- }
- statusFile.close();
- return null;
- }
- /**
- * Adds the job id to the status file.
- *
- * @param filePath the path to the status file.
- * @param jobId the job id generated following the sbatch submission.
- * @throws IOException
- */
- public static void addJobId(String filePath, String jobId) throws IOException{
- List fileContent = new ArrayList<>();
- BufferedReader br = openSourceFile(filePath);
- String line;
- while((line=br.readLine())!=null){
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) {
- line = Status.ATTRIBUTE_JOB_STATUS.getName().concat(" ").concat(Status.STATUS_JOB_RUNNING.getName());
- }
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName())) {
- line = Status.ATTRIBUTE_JOB_ID.getName().concat(" ").concat(jobId);
- }
- fileContent.add(line);
- }
- br.close();
- BufferedWriter bw = openBufferedWriter(filePath);
- for(String lineContent:fileContent){
- bw.write(lineContent.concat("\n"));
- }
- bw.flush();
- bw.close();
- }
-
- /**
- * Modifies the status of job in the status file.
- *
- * @param filePath the path to the status file.
- * @param status can be "running" or "completed".
- * @throws IOException
- */
- public static void modifyStatus(String filePath, String status) throws IOException{
- List fileContent = new ArrayList<>();
- BufferedReader br = openSourceFile(filePath);
- String line;
- while((line=br.readLine())!=null){
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) {
- line = Status.ATTRIBUTE_JOB_STATUS.getName().concat(" ").concat(status);
- }
- fileContent.add(line);
- }
- br.close();
- BufferedWriter bw = openBufferedWriter(filePath);
- for(String lineContent:fileContent){
- bw.write(lineContent.concat("\n"));
- }
- bw.flush();
- bw.close();
- }
-
- /**
- * Returns the status file if the job folder is provided.
- *
- * @param jobFolder path to the job folder.
- * @return
- */
- public static File getStatusFile(File jobFolder){
- if(jobFolder.isDirectory()){
- if((new File(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())).isFile())){
- return new File(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()));
- }
- }
- return null;
- }
-
- /**
- * Returns the log file path (absolute path) on a HPC server where the
- * Slurm job is running.
- *
- * @param runningJob
- * @param userName
- * @param taskSpace
- * @param hpcAddress
- * @return
- * @throws UnknownHostException
- */
- public static String getLogFilePathOnHPC(String runningJob, String userName, File taskSpace, String hpcAddress) throws UnknownHostException{
- String jobFolderOnHPC = runningJob.replace(hpcAddress, getMachineAddress());
- String logFilePath = getJobFolderPathOnHPC(runningJob, userName, taskSpace, hpcAddress).concat("/").concat(jobFolderOnHPC)
- .concat(Status.EXTENSION_LOG_FILE.getName());
- return logFilePath;
- }
-
- /**
- * Returns the output file path (absolute path) on a HPC server where the
- * Slurm job is running.
- *
- * @param runningJob
- * @param userName
- * @param taskSpace
- * @param hpcAddress
- * @param outputFileNameWithExtension
- * @return
- * @throws UnknownHostException
- */
- public static String getOutputFilePathOnHPC(String runningJob, String userName, File taskSpace, String hpcAddress, String outputFileNameWithExtension) throws UnknownHostException{
- String outputFilePath = getJobFolderPathOnHPC(runningJob, userName, taskSpace, hpcAddress).concat("/").concat(outputFileNameWithExtension);
- return outputFilePath;
- }
-
- /**
- * Returns the job folder path (absolute path) on a HPC where the job is running.
- *
- * @param runningJob
- * @param userName
- * @param taskSpace
- * @param hpcAddress
- * @return
- * @throws UnknownHostException
- */
- public static String getJobFolderPathOnHPC(String runningJob, String userName, File taskSpace, String hpcAddress) throws UnknownHostException{
- String jobFolderOnHPC = runningJob.replace(hpcAddress, getMachineAddress());
- String jobFolderPath = "/home/".concat(userName).concat("/")
- .concat(taskSpace.getName()).concat("/")
- .concat(jobFolderOnHPC);
- return jobFolderPath;
- }
-
- /**
- * Returns the absolute path to the output file on the machine where the Agent is running.
- *
- * @param runningJob
- * @param taskSpace
- * @param outputFileName
- * @param outputFileExtension
- * @return
- */
- public static String getJobOutputFilePathOnAgentPC(String runningJob, File taskSpace, String outputFileName, String outputFileExtension){
- return taskSpace.getAbsolutePath().concat(File.separator).concat(runningJob).concat(File.separator)
- .concat(outputFileName).concat(outputFileExtension);
- }
-
- /**
- * Checks the log file, in the case of the Gaussian software, if it is an
- * error termination.
- *
- * @param logFileOnAgentPC
- * @return
- * @throws IOException
- */
- public static boolean isErrorTermination(String logFileOnAgentPC)throws IOException{
- BufferedReader logFile = openSourceFile(logFileOnAgentPC);
- String line;
- while((line=logFile.readLine())!=null){
- if(line.trim().toLowerCase().startsWith(Status.JOB_LOG_MSG_ERROR_TERMINATION.getName().toLowerCase())){
- logFile.close();
- return true;
- }
- }
- logFile.close();
- return false;
- }
-
- /**
- * Reads the address of the machine where an instance of the agent
- * running Slurm jobs is hosted.
- *
- * @return
- * @throws UnknownHostException
- */
- public static String getMachineAddress() throws UnknownHostException{
- try {
- InetAddress address;
- address = InetAddress.getLocalHost();
- return address.toString().replace("/", "_");
- } catch (UnknownHostException e){
- e.printStackTrace();
- LOGGER.error("SlurmJobAPI: The host address is unknown as InetAddress.getLocalHost() threw an exception.");
- }
- return null;
- }
-
- /**
- * Windows uses both carriage return (\r) and line feed (\n) as a line
- * ending (\r\n). Unix uses line feed (\n) as a line ending.
- * This method translates all line endings codified with "\r\n" in a
- * file into "\n".
- *
- * @param file
- */
- public static void translateLineEndingIntoUnix(File file) throws IOException{
- File recreatedFile = new File(System.getProperty("user.home").replace("\\", "/").concat("/").concat(file.getName()));
- copyModifiedContentForUnix(file, recreatedFile);
- copyModifiedContentForUnix(recreatedFile, file);
- }
-
- /**
- * Copies file from the source path to the destination path with the
- * line ending modified to format in Unix.
- *
- * @param source
- * @param destination
- * @throws IOException
- */
- private static void copyModifiedContentForUnix(File source, File destination) throws IOException{
- BufferedReader receivedFile = openSourceFile(source.getAbsolutePath());
- BufferedWriter recreatedFile = openBufferedWriter(destination.getAbsolutePath());
- String line;
- while((line=receivedFile.readLine())!=null){
- recreatedFile.write(line.concat("\n"));
- }
- recreatedFile.close();
- receivedFile.close();
- }
-
- /**
- * Moves the provided job folder to the folder that contains completed
- * and post-processed jobs of the agent.
- *
- * @param jobFolder the folder that contains a job
- * @param slurmJobProperty
- */
- public static void moveToCompletedJobsFolder(File jobFolder, SlurmJobProperty slurmJobProperty) {
- try {
- File destDir = getCompletedJobsDirectory(jobFolder, slurmJobProperty);
- if(destDir!=null){
- FileUtils.copyDirectory(jobFolder, destDir);
- FileUtils.deleteDirectory(jobFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- /**
- * Returns the folder where completed jobs are saved.
- *
- * @param jobFolder
- * @param slurmJobProperty
- * @return
- * @throws IOException
- */
- public static File getCompletedJobsDirectory(File jobFolder, SlurmJobProperty slurmJobProperty) throws IOException{
- File workspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), slurmJobProperty.getAgentClass());
- String completedJobsDirectory = Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator).concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(workspace.getName()).concat(File.separator).concat(jobFolder.getName());
- File jobFolderInCompletedJobs = new File(completedJobsDirectory);
- jobFolderInCompletedJobs.mkdirs();
- if(jobFolderInCompletedJobs.exists()){
- return jobFolderInCompletedJobs;
- }
- return null;
- }
-
- /**
- * Moves the provided job folder to the folder that contains failed
- * jobs of the agent.
- *
- * @param jobFolder the folder that contains a job
- * @param slurmJobProperty
- */
- public static void moveToFailedJobsFolder(File jobFolder, SlurmJobProperty slurmJobProperty) {
- try {
- File destDir = getFailedJobsDirectory(jobFolder, slurmJobProperty);
- if(destDir!=null){
- FileUtils.copyDirectory(jobFolder, destDir);
- FileUtils.deleteDirectory(jobFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- /**
- * Returns the folder where failed jobs are saved.
- *
- * @param jobFolder
- * @param slurmJobProperty
- * @return
- * @throws IOException
- */
- public static File getFailedJobsDirectory(File jobFolder, SlurmJobProperty slurmJobProperty) throws IOException{
- File workspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), slurmJobProperty.getAgentClass());
- String failedJobsDirectory = Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator).concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(workspace.getName()).concat(File.separator).concat(jobFolder.getName());
- File jobFolderInFailedJobs = new File(failedJobsDirectory);
- jobFolderInFailedJobs.mkdirs();
- if(jobFolderInFailedJobs.exists()){
- return jobFolderInFailedJobs;
- }
- return null;
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Workspace.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Workspace.java
deleted file mode 100644
index 6d3e62963..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Workspace.java
+++ /dev/null
@@ -1,263 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import com.google.common.io.Files;
-
-/**
- * All functionalities for creating the workspace, job folder, status file,
- * json input file, script file and input file for Slurm jobs.
- *
- * @author msff2
- *
- */
-public class Workspace {
- private Logger LOGGER = LogManager.getLogger(Workspace.class);
- public long previousTimeStamp = System.nanoTime();
-
- /**
- * Receives both the agent class (e.g. DFTAgent) and folder (absolute path)
- * where the workspace for the agent will be created, and returns the name
- * of workspace.
- *
- * @param workspaceParentPath specific to the OS. For example,
- * - on Windows an example path will be 'C:/Users/', and
- * - on Linux the path '/home/'.
- *
- * @param agentClass
- * @return
- */
- public static File getWorkspace(String workspaceParentPath, String agentClass) {
- File dir = new File(workspaceParentPath);
- for (File file : dir.listFiles()) {
- if (file.isDirectory()) {
- if (file.getName().toLowerCase().startsWith(agentClass.toLowerCase())) {
- String[] tokens = file.getName().split("_");
- if (tokens.length >= 2 && tokens[tokens.length - 1].length() > 6
- && NumberUtils.isNumber(tokens[tokens.length - 1])) {
- return file;
- }
- }
- }
- }
- return createWorkspaceName(workspaceParentPath, agentClass);
- }
-
- private static File createWorkspaceName(String workspaceParentPath, String agentClass) {
- String workspaceName = agentClass.concat("_").concat("" + System.nanoTime());
- File workspace = Paths.get(workspaceParentPath, workspaceName).toFile();
- if (workspace.mkdir()) {
- return workspace;
- }
- return null;
- }
-
- protected boolean isWorkspaceAvailable(File dir, String agentClass) {
- for (File file : dir.listFiles()) {
- if (file.isDirectory()) {
- if (file.getName().toLowerCase().startsWith(agentClass.toLowerCase())) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Creates the JSON formatted input file in the current job folder.
- *
- * @param workspaceFolder
- * @param jsonInputFilePath the path where the JSON formatted input
- * file will be stored.
- * @param jsonString the string that contains the the whole content
- * of the JSON formatted input.
- * @return success message if the file JSON file can be created, null
- * otherwise.
- * @throws IOException
- */
- public String createJSONInputFile(File workspaceFolder, String jsonInputFilePath, String jsonString) throws IOException{
- BufferedWriter jsonInputFile = Utils.openBufferedWriter(jsonInputFilePath);
- if(jsonInputFile == null){
- return null;
- }
- jsonInputFile.write(jsonString);
- jsonInputFile.close();
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }
-
- /**
- * Returns the absolute path to an input file identified by its extension.
- * For example, .com, .g09, .csv, .jar, etc.
- *
- * @param jobFolder
- * @param hpcAddress
- * @param inputFileExtension
- * @return
- */
- public String getInputFilePath(File jobFolder, String hpcAddress, String inputFileExtension){
- return jobFolder.getAbsolutePath()
- .concat(File.separator)
- .concat(hpcAddress)
- .concat("_").concat(getTimeStampPart(jobFolder.getName()))
- .concat(inputFileExtension);
- }
-
- public String getInputFileExtension(File input) throws SlurmJobException{
-
- if(input.isFile()){
-
- return input.getAbsolutePath().substring(input.getAbsolutePath().lastIndexOf("."));
-
- }else{
-
- LOGGER.error("SlurmJobAPI: The provided input file is not a file.");
-
- throw new SlurmJobException("SlurmJobAPI: The provided input file is not a file.");
- }
- }
-
- /**
- * Creates the job folder on the host machine where the current agent runs.
- *
- * @param workspacePath
- * @param hpcAddress
- * @param timeStamp
- * @return
- */
- public File createJobFolder(String workspacePath, String hpcAddress, long timeStamp){
- String jobFolder = hpcAddress.concat("_").concat("" + timeStamp);
- File workspace = new File(workspacePath.concat(File.separator).concat(jobFolder));
- if(workspace.mkdir()){
- return workspace;
- }
- return null;
- }
-
- public String createInputFile(String inputFileDestinationPath, File input) throws IOException{
- copyFile(input, new File(inputFileDestinationPath));
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }
-
- public String createStatusFile(File workspaceFolder, String statusFilePath, String hpcAddress) throws IOException{
- BufferedWriter statusFile = Utils.openBufferedWriter(statusFilePath);
- if(statusFile == null){
- return null;
- }
- statusFile.write(Status.ATTRIBUTE_JOB_STATUS.getName().concat(" "));
- statusFile.write(Status.STATUS_JOB_NOT_STARTED.getName().concat("\n"));
- statusFile.write(Status.ATTRIBUTE_JOB_ID.getName().concat("\n"));
- statusFile.write(Status.ATTRIBUTE_AGENT_ID.getName().concat(" "));
- statusFile.write(workspaceFolder.getName().concat("\n"));
- statusFile.write(Status.ATTRIBUTE_HPC_ADDRESS.getName().concat(" "));
- statusFile.write(hpcAddress.concat("\n"));
- statusFile.write(Status.ATTRIBUTE_JOB_OUTPUT.getName().concat("\n"));
- statusFile.close();
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }
-
- public String copyFile(String source, String destination) throws IOException{
- try{
- copyFile(new File(source),
- new File(destination));
- if(destination.endsWith(".sh")){
- Utils.translateLineEndingIntoUnix(new File(destination));
- }
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }catch(IOException e){
- e.printStackTrace();
- return null;
- }
- }
-
- /**
- * Extracts the time stamp part from the job folder. For example,
- * if the job folder is named login-skylake.hpc.cam.ac.uk_428109593378500,
- * the method will return 428109593378500.
- *
- * @param folder
- * @return
- */
- public String getTimeStampPart(String folder){
- if(folder.contains("_")){
- String[] tokens = folder.split("_");
- if(tokens.length==2 && tokens[1]!=null && StringUtils.isNumeric(tokens[1])){
- return tokens[1];
- }
- }
- return null;
- }
-
- /**
- * Receives the jobFolder as the input and returns the absolute path to
- * to the job status file.
- *
- * @param jobFolder
- * @return
- */
- public String getStatusFilePath(File jobFolder){
- return jobFolder.getAbsolutePath().concat(File.separator).concat(Property.STATUS_FILE_NAME.getPropertyName());
- }
-
- /**
- * Receives the jobFolder as the input and returns the absolute path to
- * to the JSON input file.
- *
- * @param jobFolder
- * @param jsonInputFileName
- * @return
- */
- public String getJSONInputFilePath(File jobFolder){
- return jobFolder.getAbsolutePath().concat(File.separator).concat(Property.JSON_INPUT_FILE_NAME.getPropertyName());
- }
-
-
-
- /**
- * Copy the Slurm script to the job folder to set up the current
- * job. If the method can copy the script, it returns a job set up
- * success message. Otherwise, it returns null.
- *
- *
- * @param source
- * @param destination
- * @param slurmScriptFileName
- * @return
- * @throws IOException
- */
- public String copyScriptFile(String source, String destination, String slurmScriptFileName) throws IOException{
- try{
- copyFile(new File(source),
- new File(destination.concat(File.separator)
- .concat(slurmScriptFileName)));
- Utils.translateLineEndingIntoUnix(new File(destination.concat(File.separator)
- .concat(slurmScriptFileName)));
- return Status.JOB_SETUP_SUCCESS_MSG.getName();
- }catch(IOException e){
- e.printStackTrace();
- return null;
- }
- }
-
- /**
- * Copy any file from the source path to the destination path.
- *
- * @param from the absolute path to the file which will be copied.
- * @param to the absolute path to the folder where the file will be
- * copied. If user provides absolute path including the file name,
- * probably renamed or the same as the one in the from path, the method
- * will keep the same name.
- *
- * @throws IOException
- */
- private void copyFile(File from, File to) throws IOException{
- Files.copy(from, to);
- }
-}
diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobProperty.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobProperty.java
deleted file mode 100644
index fb2cf4fdd..000000000
--- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobProperty.java
+++ /dev/null
@@ -1,181 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job.configuration;
-
-/**
- * These properties must be set by all agents that will run Slurm jobs
- * with this API.
- *
- * @author Feroz Farazi (msff2@cam.ac.uk)
- *
- */
-public class SlurmJobProperty {
- private String hpcServerLoginUserName;
- private String hpcServerLoginUserPassword;
- private String hpcServerPrivateKey;
- private String agentClass;
- private String agentWorkspacePrefix;
- private String agentCompletedJobsSpacePrefix;
- private String agentFailedJobsSpacePrefix;
- private String hpcAddress;
- private String inputFileName;
- private String inputFileExtension;
- private String jsonInputFileName;
- private String jsonFileExtension;
- private String slurmScriptFileName;
- private String executableFile;
- private String outputFileName;
- private String outputFileExtension;
- private int maxNumberOfHPCJobs;
- private int agentInitialDelayToStartJobMonitoring;
- private int agentPeriodicActionInterval;
-
- public String getHpcServerLoginUserName() {
- return hpcServerLoginUserName;
- }
-
- public void setHpcServerLoginUserName(String hpcServerLoginUserName) {
- this.hpcServerLoginUserName = hpcServerLoginUserName;
- }
-
- public String getHpcServerLoginUserPassword() {
- return hpcServerLoginUserPassword;
- }
-
- public void setHpcServerLoginUserPassword(String hpcServerLoginUserPassword) {
- this.hpcServerLoginUserPassword = hpcServerLoginUserPassword;
- }
-
- public String getHpcServerPrivateKey() {
- return this.hpcServerPrivateKey;
- }
- public void setHpcServerPrivateKey(String hpcServerPrivateKey) {
- this.hpcServerPrivateKey = hpcServerPrivateKey;
- }
-
- public String getAgentClass() {
- return agentClass;
- }
-
- public String getAgentWorkspacePrefix() {
- return agentWorkspacePrefix;
- }
-
- public String getAgentCompletedJobsSpacePrefix() {
- return agentCompletedJobsSpacePrefix;
- }
-
- public String getHpcAddress() {
- return hpcAddress;
- }
-
- public String getInputFileName() {
- return inputFileName;
- }
-
- public String getInputFileExtension() {
- return inputFileExtension;
- }
-
- public String getJsonFileExtension() {
- return jsonFileExtension;
- }
-
- public String getJsonInputFileName() {
- return jsonInputFileName;
- }
-
- public String getSlurmScriptFileName() {
- return slurmScriptFileName;
- }
-
- public String getOutputFileName() {
- return outputFileName;
- }
-
- public String getOutputFileExtension() {
- return outputFileExtension;
- }
-
- public String getExecutableFile() {
- return executableFile;
- }
-
- public void setExecutableFile(String executableFile) {
- this.executableFile = executableFile;
- }
-
- public int getMaxNumberOfHPCJobs() {
- return maxNumberOfHPCJobs;
- }
-
- public int getAgentInitialDelayToStartJobMonitoring() {
- return agentInitialDelayToStartJobMonitoring;
- }
-
- public int getAgentPeriodicActionInterval() {
- return agentPeriodicActionInterval;
- }
-
- public void setAgentClass(String agentClass) {
- this.agentClass = agentClass;
- }
-
- public void setAgentWorkspacePrefix(String agentWorkspacePrefix) {
- this.agentWorkspacePrefix = agentWorkspacePrefix;
- }
-
- public void setAgentCompletedJobsSpacePrefix(String agentCompletedJobsSpacePrefix) {
- this.agentCompletedJobsSpacePrefix = agentCompletedJobsSpacePrefix;
- }
-
- public void setHpcAddress(String hpcAddress) {
- this.hpcAddress = hpcAddress;
- }
-
- public void setInputFileName(String inputFileName) {
- this.inputFileName = inputFileName;
- }
-
- public void setInputFileExtension(String inputFileExtension) {
- this.inputFileExtension = inputFileExtension;
- }
-
- public void setJsonFileExtension(String jsonFileExtension) {
- this.jsonFileExtension = jsonFileExtension;
- }
-
- public void setJsonInputFileName(String jsonInputFileName) {
- this.jsonInputFileName = jsonInputFileName;
- }
-
- public void setSlurmScriptFileName(String slurmScriptFileName) {
- this.slurmScriptFileName = slurmScriptFileName;
- }
-
- public void setOutputFileName(String outputFileName) {
- this.outputFileName = outputFileName;
- }
-
- public void setOutputFileExtension(String outputFileExtension) {
- this.outputFileExtension = outputFileExtension;
- }
-
- public void setMaxNumberOfHPCJobs(int maxNumberOfHPCJobs) {
- this.maxNumberOfHPCJobs = maxNumberOfHPCJobs;
- }
-
- public void setAgentInitialDelayToStartJobMonitoring(int agentInitialDelayToStartJobMonitoring) {
- this.agentInitialDelayToStartJobMonitoring = agentInitialDelayToStartJobMonitoring;
- }
-
- public void setAgentPeriodicActionInterval(int agentPeriodicActionInterval) {
- this.agentPeriodicActionInterval = agentPeriodicActionInterval;
- }
-
- public String getAgentFailedJobsSpacePrefix() {
- return agentFailedJobsSpacePrefix;
- }
-
- public void setAgentFailedJobsSpacePrefix(String agentFailedJobsSpacePrefix) {
- this.agentFailedJobsSpacePrefix = agentFailedJobsSpacePrefix;
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatisticsTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatisticsTest.java
deleted file mode 100644
index 8e4800ba6..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatisticsTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-class JobStatisticsTest {
- @TempDir()
- static Path tempFolder;
- private static Path fileloc;
-
- // Create JobstatisticsTestInputstatus.txt. This file will be deleted at the end
- // of the unit test.
- @BeforeAll
- static void generatefile() throws IOException {
- Files.createDirectories(tempFolder);
- fileloc = tempFolder.resolve("jobFolder").resolve("JobStatisticsTestInputstatus.txt");
- File tempInputStatus = fileloc.toFile();
- tempInputStatus.getParentFile().mkdirs();
- try (FileWriter writer = new FileWriter(tempInputStatus)) {
-
- // Fill JobStatisticsTestInputstatus.txt with text. We have one JobStatus:
- // completed, two JobStatus: completing, etc.
- for (int i = 0; i < 1; i++) {
- writer.write("JobStatus: completed\n");
- }
-
- for (int i = 0; i < 2; i++) {
- writer.write("JobStatus: completing\n");
- }
-
- for (int i = 0; i < 3; i++) {
- writer.write("JobStatus: failed\n");
- }
-
- for (int i = 0; i < 4; i++) {
- writer.write("JobStatus: pending\n");
- }
-
- for (int i = 0; i < 5; i++) {
- writer.write("JobStatus: preempted\n");
- }
-
- for (int i = 0; i < 6; i++) {
- writer.write("JobStatus: running\n");
- }
-
- for (int i = 0; i < 7; i++) {
- writer.write("JobStatus: suspended\n");
- }
-
- for (int i = 0; i < 8; i++) {
- writer.write("JobStatus: stopped\n");
- }
-
- for (int i = 0; i < 9; i++) {
- writer.write("JobStatus: error termination\n");
- }
-
- for (int i = 0; i < 10; i++) {
- writer.write("JobStatus: not started\n");
- }
- }
- }
-
- @Test
- @DisplayName("Testing calculateStatistics")
- void calculateStatistics() throws IOException, URISyntaxException {
- // Call JobstatisticsTestInputstatus.txt, which was created by generatefile().
- // This file will be deleted at the end of the unit test.
- File tempInputStatus = fileloc.toFile();
- File input = tempInputStatus.getParentFile();
-
- // Create new object And feed JobStatisticsTestInputstatus.txt as the input. The
- // file has 1 job completed, 2 jobs completing, etc, hence all of the
- // assertEquals must be true for the test to be successful
- JobStatistics result = new JobStatistics(input);
- result.calculateStatistics(fileloc.toString());
-
- assertEquals(1, result.getJobsCompleted());
- assertEquals(2, result.getJobsCompleting());
- assertEquals(3, result.getJobsFailed());
- assertEquals(4, result.getJobsPending());
- assertEquals(5, result.getJobsPreempted());
- assertEquals(6, result.getJobsRunning());
- assertEquals(7, result.getJobsSuspended());
- assertEquals(8, result.getJobsStopped());
- assertEquals(9, result.getJobsErrorTerminated());
- assertEquals(10, result.getJobsNotStarted());
- }
-
- @Test
- @DisplayName("Testing JobStatistics")
- void JobStatistics() throws IOException {
-
- File tempInputStatus = fileloc.toFile();
- File input = tempInputStatus.getParentFile().getParentFile();
-
- JobStatistics result = new JobStatistics(input);
- // There are 55 jobs total (= 1 + 2 + 3 + 4 + ... + 10). 1 completed, 2
- // completing, 3 failed, 4 pending, etc. getJobsSubmitted should therefore
- // return 55.
- assertEquals(55, result.getJobsSubmitted());
- }
-}
\ No newline at end of file
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessingTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessingTest.java
deleted file mode 100644
index 6278e1680..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessingTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-class PostProcessingTest {
- private Path tempDir;
-
- @BeforeEach
- void createTempDir() throws IOException {
- // This directory will be deleted at the end of each test.
- tempDir = Files.createTempDirectory("PostProcessingTest");
- }
-
- @AfterEach
- void deleteTempDir() throws IOException {
- if (null != tempDir) {
- FileUtils.forceDelete(tempDir.toFile());
- }
- }
-
- @Test
- void testupdateJobOutputStatus() throws IOException {
- // Create the status.txt file.
- Path tempStatusFile = tempDir.resolve("jobDir").resolve("status.txt");
- Files.createDirectories(tempStatusFile.getParent());
- Files.write(tempStatusFile, "JobOutput: processed".getBytes());
-
- // Not a valid job folder as not "status.txt" file.
- File falseDir = tempDir.toFile();
- assertFalse(PostProcessing.updateJobOutputStatus(falseDir));
- // "status.txt" file should be detected.
- File trueDir = tempStatusFile.toFile().getParentFile();
- assertTrue(PostProcessing.updateJobOutputStatus(trueDir));
- }
-
- @Test
- void testmodifyOutputStatus() throws IOException {
- // Create the status.txt file.
- Path tempStatusFile = tempDir.resolve("jobDir").resolve("status.txt");
- Files.createDirectories(tempStatusFile.getParent());
- Files.write(tempStatusFile, "JobOutput: processed".getBytes());
-
- String status = "Try!!";
- PostProcessing.modifyOutputStatus(tempStatusFile.toString(), status);
- Files.readAllLines(tempStatusFile).forEach(line -> assertEquals("JobOutput: " + status, line));
- }
-}
\ No newline at end of file
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PropertyTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PropertyTest.java
deleted file mode 100644
index 442a1a232..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PropertyTest.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-import junit.framework.TestCase;
-
-public class PropertyTest extends TestCase{
-
- // test getPropertyName
- public void testgetPropertyName() {
- String userhome = System.getProperty("user.home");
-
- assertEquals(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), userhome);
- assertEquals(Property.CHK_POINT_FILE_EXTENSION.getPropertyName(),".chk");
- assertEquals(Property.STATUS_FILE_NAME.getPropertyName(),"status.txt");
- assertEquals(Property.JSON_INPUT_FILE_NAME.getPropertyName(),"input.json");
- assertEquals(Property.SLURM_SCRIPT_FILE_NAME.getPropertyName(),"Slurm.sh");
- }
-
- // test getValue
- public void testgetValue() {
- assertEquals(Property.JOB_WORKSPACE_PARENT_DIR.getValue(), 0);
- assertEquals(Property.CHK_POINT_FILE_EXTENSION.getValue(), 0);
- assertEquals(Property.STATUS_FILE_NAME.getValue(),0);
- assertEquals(Property.JSON_INPUT_FILE_NAME.getValue(), 0);
- assertEquals(Property.SLURM_SCRIPT_FILE_NAME.getValue(), 0);
- }
-
-}
\ No newline at end of file
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobExceptionTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobExceptionTest.java
deleted file mode 100644
index a87f39b54..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobExceptionTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
-import org.junit.jupiter.api.Test;
-
-public class SlurmJobExceptionTest {
-
- // Create arbitrary Throwable cause (ArithmeticException: / by zero)
- public Throwable createExceptionCause(int a, int b) {
- try {
- int c = a / b;
- } catch (ArithmeticException e) {
- return e;
- }
- return null;
- }
-
- // Set arbitrary inputs
- String message = "test message";
- Throwable cause = createExceptionCause(1, 0);
- boolean enableSuppression = false;
- boolean writableStackTrace = false;
-
- @Test
- // Test SlurmJobException() should give same results as Exception()
- public void testSlurmJobException_None() {
- SlurmJobException testNone = new SlurmJobException();
- Exception expectedNone = new Exception();
- assertEquals(expectedNone.getMessage(), testNone.getMessage());
- }
-
- @Test
- // Test SlurmJobException(message) should give same results as Exception(message)
- public void testSlurmJobException_Message() {
- SlurmJobException testMessage = new SlurmJobException(message);
- Exception expectedMessage = new Exception(message);
- assertEquals(expectedMessage.getMessage(), testMessage.getMessage());
- }
-
- @Test
- // Test SlurmJobException(message, cause) should give same results as Exception(message, cause)
- public void testSlurmJobException_MessageandCause() {
- SlurmJobException testMessageandCause = new SlurmJobException(message, cause);
- Exception expectedMessageandCause = new Exception(message, cause);
- assertEquals(expectedMessageandCause.getMessage(), testMessageandCause.getMessage());
- assertEquals(expectedMessageandCause.getCause(), testMessageandCause.getCause());
- }
-
- @Test
- // Test SlurmJobException(cause) should give same results as Exception(cause)
- public void testSlurmJobException_Cause() {
- SlurmJobException testCause = new SlurmJobException(cause);
- Exception expectedCause = new Exception(cause);
- assertEquals(expectedCause.getCause(), testCause.getCause());
- }
-
- @Test
- // Test SlurmJobException(message, cause, enableSuppression, writableStackTrace)
- public void testAll() throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
- Constructor constrct = SlurmJobException.class.getDeclaredConstructor(String.class, Throwable.class, boolean.class, boolean.class);
- constrct.setAccessible(true);
- SlurmJobException testAll = constrct.newInstance(message, cause, enableSuppression, writableStackTrace);
-
- assertEquals("test message", testAll.getMessage());
- assertEquals(cause, testAll.getCause());
-
- // enableSuppression and writableStackTrace are false, should return zero length array
- assertEquals(0, testAll.getSuppressed().length);
- assertEquals(0, testAll.getStackTrace().length);
- }
-
-}
\ No newline at end of file
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobTest.java
deleted file mode 100644
index 29df87f8f..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobTest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
-
-public class SlurmJobTest {
-
- SlurmJob test = new SlurmJob();
-
- //Test setJobFolderName() and getJobFolderName()
- @Test
- public void testSetJobFolderName() {
- test.setJobFolderName("Example_One");
- Assert.assertEquals("Example_One", test.getJobFolderName());
- }
-
- //Test getNewJobFolderName()
- @Test
- public void testGetNewJobFolderName() {
- String folderName = "Example_Two_1234567";
- try (MockedStatic theMock = Mockito.mockStatic(Utils.class)) {
- theMock.when(Utils::getTimeStamp).thenReturn(1234567L);
- Assert.assertEquals(folderName, test.getNewJobFolderName("Example_Two"));
- }
- //Assert.assertEquals("Example_Two_".concat("" + Utils.getTimeStamp()), test.getNewJobFolderName("Example_Two"));
- }
-}
-
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/StatusTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/StatusTest.java
deleted file mode 100644
index dc6b826ca..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/StatusTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import junit.framework.TestCase;
-
-public class StatusTest extends TestCase{
-
-
- public void testStatus() {
-
-
- assertEquals(Status.STATUS_FILE.getName(),"status.txt");
- assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName(),"JobStatus:");
- assertEquals(Status.STATUS_JOB_NOT_STARTED.getName(),"not started");
- assertEquals(Status.STATUS_JOB_COMPLETED.getName(),"completed");
- assertEquals(Status.STATUS_JOB_COMPLETING.getName(),"completing");
- assertEquals(Status.STATUS_JOB_FAILED.getName(),"failed");
- assertEquals(Status.STATUS_JOB_PENDING.getName(),"pending");
- assertEquals(Status.STATUS_JOB_PREEMPTED.getName(),"preempted");
- assertEquals(Status.STATUS_JOB_RUNNING.getName(),"running");
- assertEquals(Status.STATUS_JOB_SUSPENDED.getName(),"suspended");
- assertEquals(Status.STATUS_JOB_STOPPED.getName(),"stopped");
- assertEquals(Status.STATUS_JOB_ERROR_TERMINATED.getName(),"error termination");
- assertEquals(Status.ATTRIBUTE_JOB_ID.getName(),"JobId:");
- assertEquals(Status.ATTRIBUTE_AGENT_ID.getName(),"AgentId:");
- assertEquals(Status.ATTRIBUTE_HPC_ADDRESS.getName(),"HPCAddress:");
- assertEquals(Status.ATTRIBUTE_JOB_OUTPUT.getName(),"JobOutput:");
- assertEquals(Status.OUTPUT_PROCESSED.getName(),"processed");
- assertEquals(Status.EXTENSION_SLURM_FILE.getName(),".sh");
- assertEquals(Status.EXTENSION_LOG_FILE.getName(),".log");
- assertEquals(Status.EXTENSION_JSON_FILE.getName(),".json");
- assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),"The job request has been processed successfully.");
- assertEquals(Status.JOB_SETUP_JSON_ERROR.getName(),"The JSON request is ill formatted.");
- assertEquals(Status.JOB_SETUP_SLURM_SCRIPT_ERROR.getName(),"The Slurm script could not be downloaded.");
- assertEquals(Status.JOB_SETUP_INPUT_FILE_ERROR.getName(),"The input file for the job could not be created.");
- assertEquals(Status.JOB_SETUP_ERROR.getName(),"The requested job could not be set up, therefore, resubmit the job.");
- assertEquals(Status.JOB_LOG_MSG_ERROR_TERMINATION.getName(),"error termination");
- assertEquals(Status.JOB_LOG_MSG_NORMAL_TERMINATION.getName(),"normal termination");
- assertEquals(Status.JOB_OUTPUT_FILE_EXIST_MESSAGE.getName(),"The file exist.");
- assertEquals(Status.JOB_SQUEUE_STATUS_COMPLETED.getName(),"CD");
- assertEquals(Status.JOB_SQUEUE_STATUS_COMPLETING.getName(),"CG");
- assertEquals(Status.JOB_SQUEUE_STATUS_FAILED.getName(),"F");
- assertEquals(Status.JOB_SQUEUE_STATUS_PENDING.getName(),"PD");
- assertEquals(Status.JOB_SQUEUE_STATUS_PREEMPTED.getName(),"PR");
- assertEquals(Status.JOB_SQUEUE_STATUS_RUNNING.getName(),"R");
- assertEquals(Status.JOB_SQUEUE_STATUS_SUSPENDED.getName(),"S");
- assertEquals(Status.JOB_SQUEUE_STATUS_STOPPED.getName(),"ST");
- }
-
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/UtilsTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/UtilsTest.java
deleted file mode 100644
index 91d924090..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/UtilsTest.java
+++ /dev/null
@@ -1,986 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.lang.reflect.Field;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Stream;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import uk.ac.cam.cares.jps.base.slurm.job.configuration.SlurmJobProperty;
-
-class UtilsTest {
-
- private static final String TEST_TEXT = "Lorem ipsum dolor sit amet\nconsectetur adipisci elit,\nsed eiusmod tempor incidunt\n minim veniam, quis nostrum exercitationem ullam corporis suscipit\n";
- private Path TEMP_DIR;
-
- @BeforeEach
- void createFolder() throws IOException {
- TEMP_DIR = Files.createTempDirectory("UtilsTest");
- }
-
- @AfterEach
- void cleanUp() throws IOException {
- try (Stream pathTree = Files.walk(TEMP_DIR)) {
- pathTree.sorted(Comparator.reverseOrder())
- .map(Path::toFile)
- .forEach(File::delete);
- }
- }
-
- @Test
- void getTimeStampTest() {
- assertNotEquals(Utils.getTimeStamp(), Utils.getTimeStamp());
- }
-
- @Test
- void openBufferedWriterTest() throws IOException {
-
- File tempfile = File.createTempFile("test_file", ".txt", TEMP_DIR.toFile());
-
- try (BufferedWriter bw = Utils.openBufferedWriter(tempfile.getAbsolutePath())) {
- assertNotNull(bw);
- } finally {
- Files.deleteIfExists(tempfile.toPath());
- }
- }
-
- @Test
- void openSourceFileTest() throws IOException {
- File tempfile = File.createTempFile("test_file", ".txt", TEMP_DIR.toFile());
-
- try (BufferedReader br = Utils.openSourceFile(tempfile.getAbsolutePath())) {
- assertNotNull(br);
- } finally {
- Files.deleteIfExists(tempfile.toPath());
- }
-
- }
-
- @Test
- void isJobCompletedOneTest() throws IOException {
-
- Path jobFolder = TEMP_DIR.resolve("login-skylake.hpc.cam.ac.uk_110761971919363");
- Files.createDirectories(jobFolder);
- Path statusFile = jobFolder.resolve(Status.STATUS_FILE.getName());
-
- // prepare status file for JobStatus:completed
- Files.write(statusFile,
- (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName()).getBytes());
- // test
- assertTrue(Utils.isJobCompleted(jobFolder.toFile()));
- Files.delete(statusFile);
-
- // prepare status file for JobStatus:error termination
- Files.write(statusFile,
- (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()).getBytes());
- // test
- assertTrue(Utils.isJobCompleted(jobFolder.toFile()));
- Files.delete(statusFile);
-
- // prepare status file for fail
- Files.write(statusFile, " ".getBytes());
- // test
- assertFalse(Utils.isJobCompleted(jobFolder.toFile()));
- Files.delete(statusFile);
- }
-
- @Test
- public void isJobCompletedTwoTest() throws IOException {
-
- SlurmJobProperty slurmJobProperty = new SlurmJobProperty();
- slurmJobProperty.setAgentClass("UnitTestAgent");
- slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs");
- slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs");
-
- File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentClass());
- Path jobFolder_path = taskspace.toPath().resolve("login-skylake.hpc.cam.ac.uk_110761971919363");
- Path status_path = jobFolder_path.resolve(Status.STATUS_FILE.getName());
-
- // JobStatus:completed and JobOutput:processed
- Files.createDirectories(jobFolder_path);
-
- Files.write(status_path, (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n"
- + Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName()).getBytes());
- File jobFolder = jobFolder_path.toFile();
- String[] src1 = jobFolder.list();
- File completedJobsDir = Paths.get(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentCompletedJobsSpacePrefix() + taskspace.getName(),
- jobFolder.getName())
- .toFile();
-
- assertTrue(Utils.isJobCompleted(jobFolder, slurmJobProperty));
- assertTrue(completedJobsDir.isDirectory());
- List completedFiles = Arrays.asList(completedJobsDir.list());
- for (String file : src1) {
- assertTrue(completedFiles.contains(file));
- }
- assertFalse(jobFolder.exists());
-
- // JobStatus:completed
- Files.createDirectories(jobFolder_path);
- Files.write(status_path,
- (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n").getBytes());
-
- assertTrue(Utils.isJobCompleted(jobFolder, slurmJobProperty));
- Files.delete(status_path);
-
- // JobStatus: error termination
- Files.write(status_path,
- (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()).getBytes());
- String[] src2 = jobFolder.list();
-
- File FailedJobsDir = Paths.get(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentFailedJobsSpacePrefix() + taskspace.getName(),
- jobFolder.getName())
- .toFile();
-
- assertTrue(Utils.isJobCompleted(jobFolder, slurmJobProperty));
- assertTrue(FailedJobsDir.isDirectory());
- List failedFiles = Arrays.asList(FailedJobsDir.list());
- for (String file : src2) {
- assertTrue(failedFiles.contains(file));
- }
- assertFalse(jobFolder.exists());
-
- // false scenario
- Files.createDirectories(jobFolder_path);
- Files.write(status_path, " ".getBytes());
- assertFalse(Utils.isJobCompleted(jobFolder, slurmJobProperty));
-
- FileUtils.forceDelete(taskspace);
- FileUtils.forceDelete(completedJobsDir.getParentFile());
- FileUtils.forceDelete(FailedJobsDir.getParentFile());
- }
-
- @Test
- void isJobErroneouslyCompletedOneTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for Jobstatus:error termination
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobErroneouslyCompleted(jobFolder));
- statusfile1.delete();
-
- // prepare status file for fail
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(" ");
- bw2.close();
- // test
- assertFalse(Utils.isJobErroneouslyCompleted(jobFolder));
-
- FileUtils.forceDelete(taskspace);
-
- }
-
- @Test
- void isJobErroneouslyCompletedTwoTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for Jobstatus:error termination
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobErroneouslyCompleted(status_path));
- statusfile1.delete();
-
- // prepare status file for fail
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(" ");
- bw2.close();
- // test
- assertFalse(Utils.isJobErroneouslyCompleted(status_path));
-
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void isJobRunningOneTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for JobStatus:running
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_RUNNING.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobRunning(jobFolder));
- statusfile1.delete();
-
- // prepare status file for JobStatus:completing
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETING.getName());
- bw2.close();
- // test
- assertTrue(Utils.isJobRunning(jobFolder));
- statusfile2.delete();
-
- // prepare status file for JobStatus:pending
- File statusfile3 = new File(status_path);
- BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3));
- bw3.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_PENDING.getName());
- bw3.close();
- // test
- assertTrue(Utils.isJobRunning(jobFolder));
- statusfile3.delete();
-
- // prepare status file for fail
- File statusfile4 = new File(status_path);
- BufferedWriter bw4 = new BufferedWriter(new FileWriter(statusfile4));
- bw4.write(" ");
- bw4.close();
- // test
- assertFalse(Utils.isJobRunning(jobFolder));
-
- FileUtils.forceDelete(taskspace);
-
- }
-
- @Test
- void isJobRunningTwoTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for JobStatus:running
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_RUNNING.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobRunning(status_path));
- statusfile1.delete();
-
- // prepare status file for JobStatus:completing
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETING.getName());
- bw2.close();
- // test
- assertTrue(Utils.isJobRunning(status_path));
- statusfile2.delete();
-
- // prepare status file for JobStatus:pending
- File statusfile3 = new File(status_path);
- BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3));
- bw3.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_PENDING.getName());
- bw3.close();
- // test
- assertTrue(Utils.isJobRunning(status_path));
- statusfile3.delete();
-
- // prepare status file for fail
- File statusfile4 = new File(status_path);
- BufferedWriter bw4 = new BufferedWriter(new FileWriter(statusfile4));
- bw4.write(" ");
- bw4.close();
- // test
- assertFalse(Utils.isJobRunning(status_path));
-
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void isJobNotStartedOneTest() throws IOException, NoSuchFieldException, IllegalAccessException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for JobStatus:not started
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_NOT_STARTED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobNotStarted(jobFolder));
- statusfile1.delete();
-
- // False scenario - 1
- Utils utils = new Utils();
- Field isStatusFileOpen = utils.getClass().getDeclaredField("isStatusFileOpen");
- isStatusFileOpen.setAccessible(true);
- isStatusFileOpen.setBoolean(isStatusFileOpen, true);
- assertFalse(Utils.isJobNotStarted(jobFolder));
- isStatusFileOpen.setBoolean(isStatusFileOpen, false);
-
- // False scenario - 2
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(" ");
- bw2.close();
- // test
- assertFalse(Utils.isJobNotStarted(jobFolder));
-
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void isJobNotStartedTwoTest() throws IOException, NoSuchFieldException, IllegalAccessException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for JobStatus:not started
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_NOT_STARTED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobNotStarted(status_path));
- statusfile1.delete();
-
- // False scenario 1
- Utils utils = new Utils();
- Field isStatusFileOpen = utils.getClass().getDeclaredField("isStatusFileOpen");
- isStatusFileOpen.setAccessible(true);
- isStatusFileOpen.setBoolean(isStatusFileOpen, true);
- assertFalse(Utils.isJobNotStarted(status_path));
- isStatusFileOpen.setBoolean(isStatusFileOpen, false);
-
- // False scenario 2
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(" ");
- bw2.close();
- // test
- assertFalse(Utils.isJobNotStarted(status_path));
-
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void isJobFinishedOneTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for JobStatus:completed
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobFinished(jobFolder, status_path));
- statusfile1.delete();
-
- // prepare status file for JobStatus:error termination
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName());
- bw2.close();
- // test
- assertTrue(Utils.isJobFinished(jobFolder, status_path));
- statusfile2.delete();
-
- // prepare status file for fail
- File statusfile3 = new File(status_path);
- BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3));
- bw3.write(" ");
- bw3.close();
- // test
- assertFalse(Utils.isJobFinished(jobFolder, status_path));
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void isJobFinishedTwoTest() throws IOException {
-
- SlurmJobProperty slurmJobProperty = new SlurmJobProperty();
- slurmJobProperty.setAgentClass("UnitTestAgent");
- slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs");
- slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs");
-
- File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentClass());
- ;
- String jobFolder_path = taskspace.getAbsolutePath() + File.separator
- + "login-skylake.hpc.cam.ac.uk_110761971919363";
- String status_path = jobFolder_path + File.separator + Status.STATUS_FILE.getName();
-
- // JobStatus:completed and JobOutput:processed
- File jobFolder1 = new File(jobFolder_path);
- jobFolder1.mkdirs();
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n"
- + Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName());
- bw1.close();
- String[] src1 = jobFolder1.list();
-
- File completedJobsDir_parent = new File(
- Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator)
- .concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(taskspace.getName()));
- File completedJobsDir = new File(completedJobsDir_parent + File.separator + jobFolder1.getName());
-
- assertTrue(Utils.isJobFinished(jobFolder1, status_path, slurmJobProperty));
- assertTrue(completedJobsDir.isDirectory());
- for (String file : src1) {
- assertTrue(Arrays.asList(completedJobsDir.list()).contains(file));
- }
- assertFalse(jobFolder1.exists());
-
- // JobStatus:completed
- File jobFolder2 = new File(jobFolder_path);
- jobFolder2.mkdirs();
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n");
- bw2.close();
-
- assertTrue(Utils.isJobFinished(jobFolder2, status_path, slurmJobProperty));
- statusfile2.delete();
-
- // JobStatus: error termination
- File statusfile3 = new File(status_path);
- BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3));
- bw3.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName());
- bw3.close();
- String[] src2 = jobFolder2.list();
-
- File FailedJobsDir_parent = new File(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator)
- .concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(taskspace.getName()));
- File FailedJobsDir = new File(FailedJobsDir_parent + File.separator + jobFolder2.getName());
-
- assertTrue(Utils.isJobFinished(jobFolder2, status_path, slurmJobProperty));
- assertTrue(FailedJobsDir.isDirectory());
- for (String file : src2) {
- assertTrue(Arrays.asList(FailedJobsDir.list()).contains(file));
- }
- assertFalse(jobFolder2.exists());
-
- // false scenario
- File jobFolder3 = new File(jobFolder_path);
- jobFolder3.mkdirs();
- File statusfile4 = new File(status_path);
- BufferedWriter bw4 = new BufferedWriter(new FileWriter(statusfile4));
- bw4.write(" ");
- bw4.close();
- assertFalse(Utils.isJobFinished(jobFolder3, status_path, slurmJobProperty));
-
- FileUtils.forceDelete(taskspace);
- FileUtils.forceDelete(completedJobsDir_parent);
- FileUtils.forceDelete(FailedJobsDir_parent);
-
- }
-
- @Test
- void isJobPostProcessedTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for Jobstatus:error termination
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobPostProcessed(jobFolder, status_path));
- statusfile1.delete();
-
- // prepare status file for fail
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(" ");
- bw2.close();
- // test
- assertFalse(Utils.isJobPostProcessed(jobFolder, status_path));
-
- FileUtils.forceDelete(taskspace);
-
- }
-
- @Test
- void isJobOutputProcessedOneTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for JobOutput:processed
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobOutputProcessed(jobFolder));
- statusfile1.delete();
-
- // prepare status file for fail
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(" ");
- bw2.close();
- // test
- assertFalse(Utils.isJobOutputProcessed(jobFolder));
-
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void isJobOutputProcessedTwoTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- // prepare status file for JobOutput:processed
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName());
- bw1.close();
- // test
- assertTrue(Utils.isJobOutputProcessed(status_path));
- statusfile1.delete();
-
- // prepare status file for fail
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(" ");
- bw2.close();
- // test
- assertFalse(Utils.isJobOutputProcessed(status_path));
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void getJobIdTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
- String tmpstr = Status.ATTRIBUTE_JOB_ID.getName() + " 28082086";
-
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(tmpstr);
- bw1.close();
-
- String tmpid = null;
- // extract job id
- String tokens[] = tmpstr.trim().toLowerCase().split(":");
- if (tokens.length >= 2 && tokens[1].trim().length() > 0)
- tmpid = tokens[1].trim();
-
- assertEquals(tmpid, Utils.getJobId(status_path));
- statusfile1.delete();
-
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(Status.ATTRIBUTE_JOB_ID.getName());
- bw2.close();
-
- assertNull(Utils.getJobId(status_path));
-
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void addJobIdTest() throws IOException {
-
- Path taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609");
- File jobFolder = taskspace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363").toFile();
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
- String jobid = "28082086";
-
- // prepare status file for addobId method with JobStatus and JobId
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + "\n" + Status.ATTRIBUTE_JOB_ID.getName());
- bw1.close();
-
- Utils.addJobId(status_path, jobid);
-
- // test contents of status file
- BufferedReader br1 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8"));
- String line;
- while ((line = br1.readLine()) != null) {
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + Status.STATUS_JOB_RUNNING.getName(), line);
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_ID.getName() + " " + jobid, line);
-
- }
- br1.close();
- statusfile1.delete();
-
- // prepare status file for addobId method with only JobId
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(Status.ATTRIBUTE_JOB_ID.getName());
- bw2.close();
-
- Utils.addJobId(status_path, jobid);
-
- // test contents of status file
- BufferedReader br2 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8"));
- while ((line = br2.readLine()) != null) {
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + Status.STATUS_JOB_RUNNING.getName(), line);
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_ID.getName() + " " + jobid, line);
- }
- br2.close();
- statusfile2.delete();
-
- // prepare status file for addobId method without anything
- File statusfile3 = new File(status_path);
- BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3));
- bw3.close();
-
- Utils.addJobId(status_path, jobid);
-
- // test contents of status file
- BufferedReader br3 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8"));
- while ((line = br3.readLine()) != null) {
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + Status.STATUS_JOB_RUNNING.getName(), line);
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_ID.getName() + " " + jobid, line);
- }
- br3.close();
- }
-
- @Test
- void modifyStatusTest() throws IOException {
-
- Path taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609");
- File jobFolder = taskspace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363").toFile();
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
- String status1 = Status.STATUS_JOB_RUNNING.getName();
- String status2 = Status.STATUS_JOB_COMPLETED.getName();
-
- // test for modifyStatus completed -> running
- File statusfile1 = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1));
- bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + status2 + "\n");
- bw1.close();
-
- Utils.modifyStatus(status_path, status1);
-
- BufferedReader br1 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8"));
- String line;
- while ((line = br1.readLine()) != null) {
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + status1, line);
-
- }
- br1.close();
- statusfile1.delete();
-
- // test for modifyStatus running -> completed
- File statusfile2 = new File(status_path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2));
- bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + status1 + "\n");
- bw2.close();
-
- Utils.modifyStatus(status_path, status2);
-
- BufferedReader br2 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8"));
- while ((line = br2.readLine()) != null) {
- if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName()))
- assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + status2, line);
-
- }
- br2.close();
- }
-
- @Test
- void getStatusFileTest() throws IOException {
-
- Path taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609");
- File jobFolder = taskspace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363").toFile();
- jobFolder.mkdirs();
- File jobFolder_null = new File("");
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- File statusfile = new File(status_path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile));
- bw1.close();
- assertEquals(status_path, Utils.getStatusFile(jobFolder).getAbsolutePath());
- assertNull(Utils.getStatusFile(jobFolder_null));
- }
-
- @Test
- void getLogFilePathOnHPCTest() throws UnknownHostException {
-
- String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363";
- String userName = "hpcServerUserName";
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- String hpcAddress = "login-skylake.hpc.cam.ac.uk";
- InetAddress address;
- address = InetAddress.getLocalHost();
- String MachineAddress = address.toString().replace("/", "_");
-
- assertEquals("/home/".concat(userName).concat("/").concat(taskspace.getName()).concat("/")
- .concat(runningJob.replace(hpcAddress, MachineAddress)).concat("/")
- .concat(runningJob.replace(hpcAddress, MachineAddress)).concat(Status.EXTENSION_LOG_FILE.getName()),
- Utils.getLogFilePathOnHPC(runningJob, userName, taskspace, hpcAddress));
- }
-
- @Test
- void getOutputFilePathOnHPCTest() throws UnknownHostException {
-
- String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363";
- String userName = "hpcServerUserName";
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- String hpcAddress = "login-skylake.hpc.cam.ac.uk";
- String outputFileNameWithExtension = "output.log";
- InetAddress address;
- address = InetAddress.getLocalHost();
- String MachineAddress = address.toString().replace("/", "_");
-
- assertEquals("/home/".concat(userName).concat("/").concat(taskspace.getName()).concat("/")
- .concat(runningJob.replace(hpcAddress, MachineAddress)).concat("/").concat(outputFileNameWithExtension),
- Utils.getOutputFilePathOnHPC(runningJob, userName, taskspace, hpcAddress,
- outputFileNameWithExtension));
- }
-
- @Test
- void getJobFolderPathOnHPCTest() throws UnknownHostException {
- String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363";
- String userName = "hpcServerUserName";
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- String hpcAddress = "login-skylake.hpc.cam.ac.uk";
- InetAddress address;
- address = InetAddress.getLocalHost();
- String MachineAddress = address.toString().replace("/", "_");
- assertEquals("/home/".concat(userName).concat("/").concat(taskspace.getName()).concat("/")
- .concat(runningJob.replace(hpcAddress, MachineAddress)),
- Utils.getJobFolderPathOnHPC(runningJob, userName, taskspace, hpcAddress));
-
- }
-
- @Test
- void getJobOutputFilePathOnAgentPCTest() {
-
- String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363";
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- String outputFileName = "output";
- String outputFileExtension = ".log";
- assertEquals(taskspace.getAbsolutePath().concat(File.separator).concat(runningJob).concat(File.separator)
- .concat(outputFileName).concat(outputFileExtension),
- Utils.getJobOutputFilePathOnAgentPC(runningJob, taskspace, outputFileName, outputFileExtension));
- }
-
- @Test
- void isErrorTerminationTest() throws IOException {
-
- File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile();
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String path = jobFolder.getAbsolutePath() + File.separator + "logFileOnAgentPC";
-
- File logfile1 = new File(path);
- BufferedWriter bw1 = new BufferedWriter(new FileWriter(logfile1));
- bw1.write(Status.JOB_LOG_MSG_ERROR_TERMINATION.getName().toLowerCase());
- bw1.close();
- assertTrue(Utils.isErrorTermination(path));
- logfile1.delete();
-
- File logfile2 = new File(path);
- BufferedWriter bw2 = new BufferedWriter(new FileWriter(logfile2));
- bw2.write(" ");
- bw2.close();
- assertFalse(Utils.isErrorTermination(path));
-
- FileUtils.forceDelete(taskspace);
- }
-
- @Test
- void getMachineAddressTest() throws UnknownHostException {
- InetAddress address;
- address = InetAddress.getLocalHost();
- String MachineAddress = address.toString().replace("/", "_");
- assertEquals(MachineAddress, Utils.getMachineAddress());
- }
-
- @Test
- void translateLineEndingIntoUnixTest() throws IOException {
- Path src = TEMP_DIR.resolve("src.txt");
- Path dest = TEMP_DIR.resolve("dest.txt");
-
- Files.write(src, TEST_TEXT.replace("\n", "\r\n").getBytes());
- Files.write(dest, TEST_TEXT.getBytes());
-
- Utils.translateLineEndingIntoUnix(src.toFile());
-
- assertTrue(FileUtils.contentEquals(dest.toFile(), src.toFile()));
- }
-
- @Test
- void moveToCompletedJobsFolderTest() throws IOException {
-
- SlurmJobProperty slurmJobProperty = new SlurmJobProperty();
- slurmJobProperty.setAgentClass("UnitTestAgent");
- slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs");
- slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs");
-
- File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentClass());
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- File statusfile = new File(status_path);
- BufferedWriter bw = new BufferedWriter(new FileWriter(statusfile));
- bw.write("XYZ");
- bw.close();
- String[] src = jobFolder.list();
-
- File completedJobsDir_parent = new File(
- Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator)
- .concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(taskspace.getName()));
- File completedJobsDir = new File(completedJobsDir_parent + File.separator + jobFolder.getName());
-
- Utils.moveToCompletedJobsFolder(jobFolder, slurmJobProperty);
- assertTrue(completedJobsDir.isDirectory());
- for (String file : src) {
- assertTrue(Arrays.asList(completedJobsDir.list()).contains(file));
- }
- assertFalse(jobFolder.exists());
-
- FileUtils.forceDelete(taskspace);
- FileUtils.forceDelete(completedJobsDir_parent);
- }
-
- @Test
- void getCompletedJobsDirectoryTest() throws IOException {
-
- SlurmJobProperty slurmJobProperty = new SlurmJobProperty();
- slurmJobProperty.setAgentClass("UnitTestAgent");
- slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs");
- slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs");
-
- File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentClass());
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- File completedJobsDir_parent = new File(
- Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator)
- .concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(taskspace.getName()));
- File completedJobsDir = new File(completedJobsDir_parent + File.separator + jobFolder.getName());
-
- assertEquals(completedJobsDir, Utils.getCompletedJobsDirectory(jobFolder, slurmJobProperty));
- FileUtils.forceDelete(taskspace);
- FileUtils.forceDelete(completedJobsDir_parent);
- }
-
- @Test
- void moveToFailedJobsFolderTest() throws IOException {
-
- SlurmJobProperty slurmJobProperty = new SlurmJobProperty();
- slurmJobProperty.setAgentClass("UnitTestAgent");
- slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs");
- slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs");
-
- File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentClass());
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName();
-
- File statusfile = new File(status_path);
- BufferedWriter bw = new BufferedWriter(new FileWriter(statusfile));
- bw.write("XYZ");
- bw.close();
- String[] src = jobFolder.list();
-
- File FailedJobsDir_parent = new File(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator)
- .concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(taskspace.getName()));
- File FailedJobsDir = new File(FailedJobsDir_parent + File.separator + jobFolder.getName());
-
- Utils.moveToFailedJobsFolder(jobFolder, slurmJobProperty);
- assertTrue(FailedJobsDir.isDirectory());
- for (String file : src) {
- assertTrue(Arrays.asList(FailedJobsDir.list()).contains(file));
- }
- assertFalse(jobFolder.exists());
-
- FileUtils.forceDelete(taskspace);
- FileUtils.forceDelete(FailedJobsDir_parent);
-
- }
-
- @Test
- void getFailedJobsDirectoryTest() throws IOException {
-
- SlurmJobProperty slurmJobProperty = new SlurmJobProperty();
- slurmJobProperty.setAgentClass("UnitTestAgent");
- slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs");
- slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs");
-
- File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(),
- slurmJobProperty.getAgentClass());
- File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363");
- jobFolder.mkdirs();
- File FailedJobsDir_parent = new File(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator)
- .concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(taskspace.getName()));
- File FailedJobsDir = new File(FailedJobsDir_parent + File.separator + jobFolder.getName());
-
- assertEquals(FailedJobsDir, Utils.getFailedJobsDirectory(jobFolder, slurmJobProperty));
- FileUtils.forceDelete(taskspace);
- FileUtils.forceDelete(FailedJobsDir_parent);
-
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/WorkspaceTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/WorkspaceTest.java
deleted file mode 100644
index 299d143b7..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/WorkspaceTest.java
+++ /dev/null
@@ -1,248 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Comparator;
-import java.util.stream.Stream;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-class WorkspaceTest {
-
- private static final String TEST_TEXT = "Lorem ipsum dolor sit amet\nconsectetur adipisci elit,\nsed eiusmod tempor incidunt\n minim veniam, quis nostrum exercitationem ullam corporis suscipit\n";
- private Path TEMP_DIR;
-
- @BeforeEach
- void createFolder() throws IOException {
- TEMP_DIR = Files.createTempDirectory("WorkspaceTest");
- }
-
- @AfterEach
- void cleanUp() throws IOException {
- try (Stream pathTree = Files.walk(TEMP_DIR)) {
- pathTree.sorted(Comparator.reverseOrder())
- .map(Path::toFile)
- .forEach(File::delete);
- }
- }
-
- @Test
- void getExistingWorkspaceTest() throws IOException {
- String agentClass = "UnitTestAgent";
- Path workspace = TEMP_DIR.resolve(agentClass + "_" + System.nanoTime());
- Files.createDirectories(workspace);
-
- assertEquals(workspace.toAbsolutePath(),
- Workspace.getWorkspace(TEMP_DIR.toString(), agentClass).toPath().toAbsolutePath());
-
- Files.delete(workspace);
- }
-
- @Test
- void getNewWorkspaceTest() throws IOException {
- String agentClass = "UnitTestAgent";
- assertEquals(TEMP_DIR.resolve(agentClass).toString(),
- Workspace.getWorkspace(TEMP_DIR.toString(), agentClass).getAbsolutePath().split("_")[0]);
- }
-
- @Test
- void isWorkspaceAvailableTestTwo()
- throws IllegalAccessException, InvocationTargetException, NoSuchMethodException, IOException {
-
- String agentClass = "UnitTestAgent";
- Path ws = TEMP_DIR.resolve(agentClass + "_" + System.nanoTime());
- Files.createDirectories(ws);
-
- Workspace workspace = new Workspace();
-
- assertTrue(workspace.isWorkspaceAvailable(TEMP_DIR.toFile(), agentClass));
- Files.delete(ws);
- assertFalse(workspace.isWorkspaceAvailable(TEMP_DIR.toFile(), agentClass));
- }
-
- @Test
- void createJSONInputFileTest() throws IOException {
-
- Path workspaceFolder = TEMP_DIR;
- Path jsonInputFilePath = workspaceFolder.resolve("input.json").toAbsolutePath();
- Path jsonInputFilePath_temp = workspaceFolder.resolve("input_temp.json").toAbsolutePath();
- Workspace workspace = new Workspace();
-
- Files.write(jsonInputFilePath_temp, TEST_TEXT.getBytes());
-
- assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),
- workspace.createJSONInputFile(workspaceFolder.toFile(), jsonInputFilePath.toString(), TEST_TEXT));
- assertTrue(FileUtils.contentEquals(jsonInputFilePath.toFile(), jsonInputFilePath_temp.toFile()));
- }
-
- @Test
- void getInputFilePathTest() throws IOException {
-
- Path taskSpace = TEMP_DIR.resolve("UnitTestAgent_435827288195609");
- Path jobFolder = taskSpace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363");
- Files.createDirectories(jobFolder);
- String hpcAddress = "login-skylake.hpc.cam.ac.uk";
- String inputFileExtension = ".com";
- String[] tokens = jobFolder.getFileName().toString().split("_");
- String timeStampPart = null;
- if (tokens.length == 2 && tokens[1] != null && StringUtils.isNumeric(tokens[1]))
- timeStampPart = tokens[1];
- Workspace workspace = new Workspace();
- assertEquals(
- jobFolder.resolve(hpcAddress + "_" + timeStampPart + inputFileExtension).toAbsolutePath().toString(),
- workspace.getInputFilePath(jobFolder.toFile(), hpcAddress, inputFileExtension));
- }
-
- @Test
- void getInputFileExtension() throws SlurmJobException, IOException {
-
- Path input = TEMP_DIR.resolve("input.csv");
-
- Files.createFile(input);
-
- Workspace workspace = new Workspace();
-
- assertEquals(input.toString().substring(input.toString().lastIndexOf('.')),
- workspace.getInputFileExtension(input.toFile()));
- }
-
- @Test
- void createJobFolderTest() throws IOException {
- Path workspacePath = TEMP_DIR;
- String hpcAddress = "login-skylake.hpc.cam.ac.uk";
- long timeStamp = System.nanoTime();
- Workspace workspace = new Workspace();
- Path jobFolder = workspacePath.resolve(hpcAddress + "_" + timeStamp);
-
- assertEquals(jobFolder.toAbsolutePath(),
- workspace.createJobFolder(workspacePath.toString(), hpcAddress, timeStamp).toPath().toAbsolutePath());
- assertNull(workspace.createJobFolder(workspacePath.toString(), hpcAddress, timeStamp));
- }
-
- @Test
- void createInputFileTest() throws IOException {
-
- Path inputFileDestinationPath = TEMP_DIR.resolve("input_dest");
- Path input = TEMP_DIR.resolve("input_src");
- Workspace workspace = new Workspace();
-
- Files.write(input, TEST_TEXT.getBytes());
-
- assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),
- workspace.createInputFile(inputFileDestinationPath.toString(), input.toFile()));
- assertTrue(FileUtils.contentEquals(input.toFile(), inputFileDestinationPath.toFile()));
- }
-
- @Test
- void createStatusFileTest() throws IOException {
-
- Path workspaceFolder = TEMP_DIR;
- Path statusFilePath = workspaceFolder.resolve(Status.STATUS_FILE.getName());
- String hpcAddress = "login-skylake.hpc.cam.ac.uk";
- Path tempFile = workspaceFolder.resolve("temp.txt");
- Workspace workspace = new Workspace();
-
- try (BufferedWriter bw = new BufferedWriter(new FileWriter(tempFile.toFile()))) {
- bw.write(Status.ATTRIBUTE_JOB_STATUS.getName() + " ");
- bw.write(Status.STATUS_JOB_NOT_STARTED.getName() + "\n");
- bw.write(Status.ATTRIBUTE_JOB_ID.getName() + "\n");
- bw.write(Status.ATTRIBUTE_AGENT_ID.getName() + " ");
- bw.write(workspaceFolder.getFileName() + "\n");
- bw.write(Status.ATTRIBUTE_HPC_ADDRESS.getName() + " ");
- bw.write(hpcAddress + "\n");
- bw.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + "\n");
- }
-
- assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),
- workspace.createStatusFile(workspaceFolder.toFile(), statusFilePath.toString(), hpcAddress));
- assertTrue(FileUtils.contentEquals(statusFilePath.toFile(), tempFile.toFile()));
- }
-
- @Test
- void copyFileTestOne() throws IOException {
-
- Path destination = TEMP_DIR.resolve("dest");
- Path source = TEMP_DIR.resolve("src");
- Path temp = TEMP_DIR.resolve("temp");
- Workspace workspace = new Workspace();
-
- Files.write(source, TEST_TEXT.getBytes());
-
- assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),
- workspace.copyFile(source.toString(), destination.toString()));
- assertTrue(FileUtils.contentEquals(source.toFile(), destination.toFile()));
-
- Files.delete(destination);
-
- Path destination_sh = TEMP_DIR.resolve("dest.sh");
- Files.write(temp, TEST_TEXT.getBytes());
-
- assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),
- workspace.copyFile(source.toString(), destination_sh.toString()));
- assertTrue(FileUtils.contentEquals(temp.toFile(), destination_sh.toFile()));
- }
-
- @Test
- void getTimeStampPartTest() {
-
- String folder = "login-skylake.hpc.cam.ac.uk_428109593378500";
- String folder_null = "";
- Workspace workspace = new Workspace();
- String[] tokens = folder.split("_");
- assertEquals(tokens[1], workspace.getTimeStampPart(folder));
- assertNull(workspace.getTimeStampPart(folder_null));
- }
-
- @Test
- void getStatusFilePathTest() {
-
- Path jobFolder = TEMP_DIR;
- Workspace workspace = new Workspace();
- String expected = jobFolder.resolve(Property.STATUS_FILE_NAME.getPropertyName()).toAbsolutePath().toString();
- assertEquals(expected, workspace.getStatusFilePath(jobFolder.toFile()));
- }
-
- @Test
- void getJSONInputFilePathTest() {
-
- Path jobFolder = TEMP_DIR;
- Workspace workspace = new Workspace();
- String expected = jobFolder.resolve(Property.JSON_INPUT_FILE_NAME.getPropertyName()).toAbsolutePath()
- .toString();
- assertEquals(expected, workspace.getJSONInputFilePath(jobFolder.toFile()));
- }
-
- @Test
- void copyScriptFileTest() throws IOException {
-
- Path source = TEMP_DIR.resolve("source");
- Path dest = TEMP_DIR.resolve("dest");
- Files.createDirectories(dest);
-
- String slurmScriptFileName = "slurmscript.sh";
- Path temp = TEMP_DIR.resolve("temp");
- Workspace workspace = new Workspace();
-
- Files.write(source, TEST_TEXT.getBytes());
- Files.write(temp, TEST_TEXT.getBytes());
-
- assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),
- workspace.copyScriptFile(source.toString(), dest.toString(), slurmScriptFileName));
- Path slurmScriptPath = dest.resolve(slurmScriptFileName);
- assertTrue(FileUtils.contentEquals(temp.toFile(), slurmScriptPath.toFile()));
- }
-}
diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobPropertyTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobPropertyTest.java
deleted file mode 100644
index 892a813a0..000000000
--- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobPropertyTest.java
+++ /dev/null
@@ -1,142 +0,0 @@
-package uk.ac.cam.cares.jps.base.slurm.job.configuration;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-import org.junit.Test;
-
-public class SlurmJobPropertyTest {
-
- SlurmJobProperty test = new SlurmJobProperty();
-
- @Test
- public void getHpcServerLoginUserNameTest() {
- // Default value
- assertNull(test.getHpcServerLoginUserName());
- // Set specific value and check whether it is set correctly
- test.setHpcServerLoginUserName("user");
- assertEquals("user", test.getHpcServerLoginUserName());
- }
-
- @Test
- public void getHpcServerLoginUserPasswordTest() {
- assertNull(test.getHpcServerLoginUserPassword());
- test.setHpcServerLoginUserPassword("user");
- assertEquals("user", test.getHpcServerLoginUserPassword());
- }
-
- @Test
- public void getAgentClassTest() {
- assertNull(test.getAgentClass());
- test.setAgentClass("user");
- assertEquals("user", test.getAgentClass());
- }
-
- @Test
- public void getAgentWorkspacePrefixTest() {
- assertNull(test.getAgentWorkspacePrefix());
- test.setAgentWorkspacePrefix("user");
- assertEquals("user", test.getAgentWorkspacePrefix());
- }
-
- @Test
- public void getAgentCompletedJobsSpacePrefixTest() {
- assertNull(test.getAgentCompletedJobsSpacePrefix());
- test.setAgentCompletedJobsSpacePrefix("user");
- assertEquals("user", test.getAgentCompletedJobsSpacePrefix());
- }
-
- @Test
- public void getHpcAddressTest() {
- assertNull(test.getHpcAddress());
- test.setHpcAddress("user");
- assertEquals("user", test.getHpcAddress());
- }
-
- @Test
- public void getInputFileNameTest() {
- assertNull(test.getInputFileName());
- test.setInputFileName("user");
- assertEquals("user", test.getInputFileName());
- }
-
- @Test
- public void getInputFileExtensionTest() {
- assertNull(test.getInputFileExtension());
- test.setInputFileName("user");
- assertEquals("user", test.getInputFileName());
- }
-
- @Test
- public void getJsonInputFileNameTest() {
- assertNull(test.getJsonInputFileName());
- test.setJsonInputFileName("user");
- assertEquals("user", test.getJsonInputFileName());
- }
-
- @Test
- public void getJsonFileExtensionTest() {
- assertNull(test.getJsonFileExtension());
- test.setJsonFileExtension("user");
- assertEquals("user", test.getJsonFileExtension());
- }
-
- @Test
- public void getSlurmScriptFileNameTest() {
- assertNull(test.getSlurmScriptFileName());
- test.setSlurmScriptFileName("user");
- assertEquals("user", test.getSlurmScriptFileName());
- }
-
- @Test
- public void getOutputFileNameTest() {
- assertNull(test.getOutputFileName());
- test.setOutputFileName("user");
- assertEquals("user", test.getOutputFileName());
- }
-
- @Test
- public void getOutputFileExtensionTest() {
- assertNull(test.getOutputFileExtension());
- test.setOutputFileExtension("user");
- assertEquals("user", test.getOutputFileExtension());
- }
-
- @Test
- public void getExecutableFileTest() {
- assertNull(test.getExecutableFile());
- test.setExecutableFile("user");
- assertEquals("user", test.getExecutableFile());
- }
-
- @Test
- public void getAgentFailedJobsSpacePrefixTest() {
- assertNull(test.getAgentFailedJobsSpacePrefix());
- test.setAgentFailedJobsSpacePrefix("user");
- assertEquals("user", test.getAgentFailedJobsSpacePrefix());
- }
-
-
- @Test
- public void getMaxNumberOfHPCJobsTest() {
- assertEquals(test.getMaxNumberOfHPCJobs(), 0);
- test.setMaxNumberOfHPCJobs(123);
- assertEquals(123, test.getMaxNumberOfHPCJobs());
- }
-
- @Test
- public void getAgentInitialDelayToStartJobMonitoringTest() {
- assertEquals(test.getAgentInitialDelayToStartJobMonitoring(), 0);
- test.setAgentInitialDelayToStartJobMonitoring(123);
- assertEquals(123, test.getAgentInitialDelayToStartJobMonitoring());
- }
-
- @Test
- public void getAgentPeriodicActionIntervalTest() {
- assertEquals(test.getAgentPeriodicActionInterval(), 0);
- test.setAgentPeriodicActionInterval(123);
- assertEquals(123, test.getAgentPeriodicActionInterval());
-
- }
-
-}
\ No newline at end of file