diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatistics.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatistics.java deleted file mode 100644 index 1e354ee09..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatistics.java +++ /dev/null @@ -1,312 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; - -/** - * This class implemented methods for the calculation and visualisation of
- * statistics of jobs submitted to an HPC. - * - * @author msff2 - * - */ -public class JobStatistics { - private int jobsSubmitted = 0; - private int jobsCompleted = 0; - private int jobsCompleting = 0; - private int jobsFailed = 0; - private int jobsPending = 0; - private int jobsPreempted = 0; - private int jobsRunning = 0; - private int jobsSuspended = 0; - private int jobsStopped = 0; - private int jobsErrorTerminated = 0; - private int jobsNotStarted = 0; - - private File jobSpace; - - /** - * Construct that takes the job workspace folder as the parameter and
- * calculates the statistics of jobs submitted to an HPC. - * - * @param jobSpace - * @throws IOException - */ - public JobStatistics(File jobSpace) throws IOException{ - this.jobSpace = jobSpace; - File[] jobs = jobSpace.listFiles(); - for(File job:jobs){ - calculateAllStatistics(job); - } - setJobsSubmitted(getJobsCompleted() + getJobsCompleting() + getJobsFailed() + getJobsPending() - + getJobsPreempted() + getJobsRunning() + getJobsSuspended() + getJobsStopped() - + getJobsErrorTerminated() + getJobsNotStarted()); - } - - /** - * Calculates statistics of jobs grouped into multiple categories based
- * upon their status. - * - * @param job - * @throws IOException - */ - private void calculateAllStatistics(File job) throws IOException{ - if(job!=null && job.isDirectory()){ - File[] jobFiles = job.listFiles(); - for(File jobFile:jobFiles){ - calculateStatistics(jobFile); - } - } - } - - /** - * Reads the status of single job to calculate the statistics. - * - * @param jobFile - * @throws IOException - */ - private void calculateStatistics(File jobFile) throws IOException{ - if(!jobFile.isDirectory() && jobFile.getAbsolutePath().endsWith(Status.STATUS_FILE.getName())){ - calculateStatistics(jobFile.getAbsolutePath()); - } - } - /** - * Calculates statistics about all submitted jobs. - * - * @param statusFilePath - * @throws IOException - */ - public void calculateStatistics(String statusFilePath) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){ - if(line.contains(Status.STATUS_JOB_COMPLETED.getName())){ - setJobsCompleted(getJobsCompleted()+1); - } else if(line.contains(Status.STATUS_JOB_COMPLETING.getName())){ - setJobsCompleting(getJobsCompleting()+1); - } else if(line.contains(Status.STATUS_JOB_FAILED.getName())){ - setJobsFailed(getJobsFailed()+1); - } else if(line.contains(Status.STATUS_JOB_PENDING.getName())){ - setJobsPending(getJobsPending()+1); - } else if(line.contains(Status.STATUS_JOB_PREEMPTED.getName())){ - setJobsPreempted(getJobsPreempted()+1); - } else if(line.contains(Status.STATUS_JOB_RUNNING.getName())){ - setJobsRunning(getJobsRunning()+1); - } else if(line.contains(Status.STATUS_JOB_SUSPENDED.getName())){ - setJobsSuspended(getJobsSuspended()+1); - } else if(line.contains(Status.STATUS_JOB_STOPPED.getName())){ - setJobsStopped(getJobsStopped()+1); - } else if(line.contains(Status.STATUS_JOB_ERROR_TERMINATED.getName())){ - setJobsErrorTerminated(getJobsErrorTerminated()+1); - } else if(line.contains(Status.STATUS_JOB_NOT_STARTED.getName())){ - setJobsNotStarted(getJobsNotStarted()+1); - } - } - } - statusFile.close(); - } - - public void setJobSpace(File jobSpace) { - this.jobSpace = jobSpace; - } - - public int getJobsSubmitted() { - return jobsSubmitted; - } - public void setJobsSubmitted(int jobsSubmitted) { - this.jobsSubmitted = jobsSubmitted; - } - public int getJobsRunning() { - return jobsRunning; - } - public void setJobsRunning(int jobsRunning) { - this.jobsRunning = jobsRunning; - } - public int getJobsCompleted() { - return jobsCompleted; - } - public void setJobsCompleted(int jobsCompleted) { - this.jobsCompleted = jobsCompleted; - } - public int getJobsErrorTerminated() { - return jobsErrorTerminated; - } - public void setJobsErrorTerminated(int jobsErrorTerminated) { - this.jobsErrorTerminated = jobsErrorTerminated; - } - public int getJobsNotStarted() { - return jobsNotStarted; - } - public void setJobsNotStarted(int jobsNotStarted) { - this.jobsNotStarted = jobsNotStarted; - } - - public int getJobsCompleting() { - return jobsCompleting; - } - - public void setJobsCompleting(int jobsCompleting) { - this.jobsCompleting = jobsCompleting; - } - - public int getJobsFailed() { - return jobsFailed; - } - - public void setJobsFailed(int jobsFailed) { - this.jobsFailed = jobsFailed; - } - - public int getJobsPending() { - return jobsPending; - } - - public void setJobsPending(int jobsPending) { - this.jobsPending = jobsPending; - } - - public int getJobsPreempted() { - return jobsPreempted; - } - - public void setJobsPreempted(int jobsPreempted) { - this.jobsPreempted = jobsPreempted; - } - - public int getJobsSuspended() { - return jobsSuspended; - } - - public void setJobsSuspended(int jobsSuspended) { - this.jobsSuspended = jobsSuspended; - } - - public int getJobsStopped() { - return jobsStopped; - } - - public void setJobsStopped(int jobsStopped) { - this.jobsStopped = jobsStopped; - } - - /** - * Produces the header for showing the statistics table. - * - * @return mainly the header of HTML. - */ - public String getHTMLHeader(){ - String htmlHead = ""; - htmlHead = htmlHead.concat(""); - htmlHead = htmlHead.concat(""); - htmlHead = htmlHead.concat(""); - htmlHead = htmlHead.concat(getResources()); - htmlHead = htmlHead.concat(""); - return htmlHead; - } - - /** - * Adds links to java script libraries and style sheets. - * - * @return - */ - private String getResources(){ - String resourcePath = pathToResource(); - String headerResource = ""; - headerResource = headerResource.concat(""); - headerResource = headerResource.concat(getLink("")); - headerResource = headerResource.concat(getLink("")); - headerResource = headerResource.concat(getLink("")); - headerResource = headerResource.concat(getLink("")); - headerResource = headerResource.concat("J-Park Simulator"); - return headerResource; - } - - public String getBodydivStart(){ - String resourcePath = pathToResource(); - String startDiv = "
"; - startDiv = startDiv.concat(""); - startDiv = startDiv.concat("")); - startDiv = startDiv.concat(""); - startDiv = startDiv.concat("

A JPS Agent

"); - startDiv = startDiv.concat("

"); - startDiv = startDiv.concat("An agent developed for performing simulations on HPC servers " - + "to create a living, continuously self-improving and growing " - + "Knowledge Graph of species serving the need for thermochemical data across disciplines."); - startDiv = startDiv.concat("

"); - startDiv = startDiv.concat("
"); - return startDiv; - } - - /** - * Converts path specific to a Windows or Linux server. - * - * @param resourcePath - * @return - */ - private String pathToResource(){ - return "/KineticsAgent/assets/"; - } - - /** - * Produces a link to a java script library or style sheet using the
- * link tag and the path where the resource is located. - * - * @param link - * @param hrefStartPath - * @param hrefEndPath - * @return - */ - private String getLink(String link, String hrefStartPath, String hrefEndPath){ - String linkResource = link; - linkResource = linkResource.concat(hrefStartPath); - linkResource = linkResource.concat(hrefEndPath); - return linkResource; - } - - /** - * Prepares the header of table showing statistics about jobs submitted to DFT Agent. - * - * @return mainly the header part of the HTML table showing job statistics. - */ - public String getStatisticsTableHeader(String headerText, String statisticsProperty, String statisticsValue, String tableWidth){ - String tableHeader = "

"+headerText+"


"; - tableHeader = tableHeader + ""; - tableHeader = tableHeader + ""; - tableHeader = tableHeader + ""; - tableHeader = tableHeader + ""; - tableHeader = tableHeader + ""; - return tableHeader; - } - - /** - * Prepares the row of table showing statistics about jobs submitted to DFT Agent. - * - * @param property - * @param value - * @return - */ - public String getStatisticsTableRow(String property, String value){ - String tableRow = ""; - tableRow = tableRow + ""; - tableRow = tableRow + ""; - tableRow = tableRow + ""; - tableRow = tableRow + ""; - return tableRow; - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobSubmission.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobSubmission.java deleted file mode 100644 index 8cf17befa..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/JobSubmission.java +++ /dev/null @@ -1,1117 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.json.JSONObject; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelExec; -import com.jcraft.jsch.ChannelSftp; -import com.jcraft.jsch.IdentityRepository; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.Session; -import com.jcraft.jsch.SftpException; -import com.jcraft.jsch.agentproxy.AgentProxyException; -import com.jcraft.jsch.agentproxy.Connector; -import com.jcraft.jsch.agentproxy.RemoteIdentityRepository; -import com.jcraft.jsch.agentproxy.connector.PageantConnector; - -import uk.ac.cam.cares.jps.base.slurm.job.configuration.SlurmJobProperty; - -/** - * This is an API developed for setting-up and running Slurm jobs. - * - * @author msff2 - * - */ -public class JobSubmission{ - private Logger LOGGER = LogManager.getLogger(JobSubmission.class); - private String hpcAddress; - private String agentClass; - private File workspaceDirectory; - private String workspaceName; - private String workspaceParentPath; - boolean isAuthenticated; - - public SlurmJobProperty slurmJobProperty = new SlurmJobProperty(); - - static Session session; - static JSch jsch = new JSch(); - - static int scheduledIteration = 0; - static Set jobsRunning = new HashSet(); - - public String getHpcAddress() { - return hpcAddress; - } - - public void setHpcAddress(String hpcAddress) { - this.hpcAddress = hpcAddress; - } - - public String getAgentClass() { - return agentClass; - } - - public void setAgentClass(String agentClass) { - this.agentClass = agentClass; - } - - public File getWorkspaceDirectory() { - return workspaceDirectory; - } - - public void setWorkspaceDirectory(File workspaceDirectory) { - this.workspaceDirectory = workspaceDirectory; - } - - public String getWorkspaceName() { - return workspaceName; - } - - public void setWorkspaceName(String workspaceName) { - this.workspaceName = workspaceName; - } - - public static void main(String[] args) throws SlurmJobException{ -// JobSubmission jobSubmission = new JobSubmission("DFTAgent", new File(System.getProperty("user.home")), "login-skylake.hpc.cam.ac.uk"); -// jobSubmission.init(); - } - - public String getWorkspaceParentPath() { - return workspaceParentPath; - } - - public void setWorkspaceParentPath(String workspaceParentPath) { - this.workspaceParentPath = workspaceParentPath; - } - - /** - * Constructor for this class. - * - * @param agentClass the name of the agent class (e.g. DFTAgent and EBRAgent). - * @param workspaceName the name of workspace
- * will be created, e.g. user home directory read by System.getProperty("user.home"); - * @param hpcAddress the address of HPC, e.g. login-skylake.hpc.cam.ac.uk for CSD3. - */ - public JobSubmission(String agentClass, String hpcAddress){ - this.agentClass = agentClass; - this.hpcAddress = hpcAddress; - this.workspaceDirectory = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), agentClass); - this.workspaceParentPath = Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(); - } - - /** - * Sets up a Slurm job by creating the job folder and the following files
- * under this folder:
- * - the input file.
- * - the Slurm script file. - * - the JSON input file, which comes from the user request.
- * - * @param jsonInput - * @param slurmScript - * @param input - * @param timeStamp - * @return - * @throws IOException - * @throws DFTAgentException - */ - public String setUpJob(String jsonInput, File slurmScript, File input, long timeStamp) throws IOException, SlurmJobException{ - String message = setUpJobOnAgentMachine(jsonInput, slurmScript, input, timeStamp); - return message; - } - - /** - * Sets up a Slurm job by creating the job folder and the following files
- * under this folder:
- * - the input file.
- * - the Slurm script file. - * - the JSON input file, which comes from the user request.
- * - * @param jsonInput - * @param slurmScript - * @param input - * @param executable - * @param timeStamp - * @return - * @throws IOException - * @throws DFTAgentException - */ - public String setUpJob(String jsonInput, File slurmScript, File input, File executable, long timeStamp) throws IOException, SlurmJobException{ - String message = setUpJobOnAgentMachine(jsonInput, slurmScript, input, executable, timeStamp); - JSONObject obj = new JSONObject(); - obj.put("message", message); - return obj.toString(); - } - - - /** - * Sets up the Slurm job for the current input. - * - * @param jsonString - * @param slurmScript - * @param input - * @param timeStamp - * @return - * @throws IOException - * @throws DFTAgentException - */ - private String setUpJobOnAgentMachine(String jsonString, File slurmScript, File input, long timeStamp) throws IOException, SlurmJobException{ - Workspace workspace = new Workspace(); - if(workspaceDirectory == null){ - LOGGER.error("Workspace directory has not been set!"); - return Status.JOB_SETUP_ERROR.getName(); - }else{ - return setUpSlurmJob(workspace, workspaceDirectory, jsonString, slurmScript, input, timeStamp); - } - } - - /** - * Sets up the Slurm job for the current input. - * - * @param jsonString - * @param slurmScript - * @param input - * @param executable - * @param timeStamp - * @return - * @throws IOException - * @throws DFTAgentException - */ - private String setUpJobOnAgentMachine(String jsonString, File slurmScript, File input, File executable, long timeStamp) throws IOException, SlurmJobException{ - Workspace workspace = new Workspace(); - if(workspaceDirectory == null){ - LOGGER.error("Workspace directory has not been set!"); - return Status.JOB_SETUP_ERROR.getName(); - }else{ - return setUpSlurmJob(workspace, workspaceDirectory, jsonString, slurmScript, input, executable, timeStamp); - } - } - - /** - * Sets up the Slurm job for the current request. - * - * @param ws - * @param workspaceFolder - * @param jsonString - * @param slurmScript - * @param input - * @param timeStamp - * @return - * @throws IOException - * @throws DFTAgentException - */ - private String setUpSlurmJob(Workspace ws, File workspaceFolder, String jsonString, File slurmScript, File input, long timeStamp) throws IOException, SlurmJobException{ - File jobFolder = ws.createJobFolder(workspaceFolder.getAbsolutePath(), getHpcAddress(), timeStamp); - LOGGER.info("Set up slurm job folder at: " + jobFolder); - - if(createAllFileInJobFolder(ws, workspaceFolder, jobFolder, jsonString, slurmScript, input) == null) { - LOGGER.error("Could not create files within the slurm job folder."); - return null; - } - return jobFolder.getName(); - } - - /** - * Sets up the Slurm job for the current request. - * - * @param ws - * @param workspaceFolder - * @param jsonString - * @param slurmScript - * @param input - * @param executable - * @param timeStamp - * @return - * @throws IOException - * @throws DFTAgentException - */ - private String setUpSlurmJob(Workspace ws, File workspaceFolder, String jsonString, File slurmScript, File input, File executable, long timeStamp) throws IOException, SlurmJobException{ - File jobFolder = ws.createJobFolder(workspaceFolder.getAbsolutePath(), getHpcAddress(), timeStamp); - LOGGER.info("Set up slurm job folder at: " + jobFolder); - - if(createAllFileInJobFolder(ws, workspaceFolder, jobFolder, jsonString, slurmScript, input, executable) == null) { - LOGGER.error("Could not create files within the slurm job folder."); - return null; - } - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - } - - /** - * Creates all files relevant for the current job, in particular, it
- * creates the following files:
- * - the input file in com (.com) format for running the job on an HPC - * - the status file in text (.txt) format - * - the input file in json (.json) format - * - the Slurm script file in shell script (.sh) format - * - * @param ws - * @param workspaceFolder - * @param jobFolder - * @param jsonString - * @param slurmScript - * @param input - * @return - * @throws IOException - * @throws DFTAgentException - */ - private String createAllFileInJobFolder(Workspace ws, File workspaceFolder, File jobFolder, String jsonString, File slurmScript, File input) throws IOException, SlurmJobException{ - String inputFileMsg = ws.createInputFile(ws.getInputFilePath(jobFolder, getHpcAddress(), ws.getInputFileExtension(input)), input); - if(inputFileMsg == null){ - return Status.JOB_SETUP_INPUT_FILE_ERROR.getName(); - } - String statusFileMsg = ws.createStatusFile(workspaceFolder, ws.getStatusFilePath(jobFolder), getHpcAddress()); - if(statusFileMsg == null){ - return null; - } - String jsonInputFileMsg = ws.createJSONInputFile(workspaceFolder, ws.getJSONInputFilePath(jobFolder), jsonString); - if(jsonInputFileMsg == null){ - return null; - } - String scriptFileMsg = ws.copyFile(slurmScript.getAbsolutePath(), jobFolder.getAbsolutePath().concat(File.separator).concat(slurmScript.getName())); - if(scriptFileMsg == null){ - return null; - } - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - } - - /** - * Creates all files relevant for the current job, in particular, it
- * creates the following files:
- * - the input file in zip (.zip) format for running the job on an HPC - * - the status file in text (.txt) format - * - the input file in json (.json) format - * - the Slurm script file in shell script (.sh) format - * - the executable file in java is in jar (.jar) format - * - * @param ws - * @param workspaceFolder - * @param jobFolder - * @param jsonString - * @param slurmScript - * @param input - * @param executable - * @return - * @throws IOException - * @throws DFTAgentException - */ - private String createAllFileInJobFolder(Workspace ws, File workspaceFolder, File jobFolder, String jsonString, File slurmScript, File input, File executable) throws IOException, SlurmJobException{ - String inputFileMsg = ws.createInputFile(ws.getInputFilePath(jobFolder, getHpcAddress(), ws.getInputFileExtension(input)), input); - if(inputFileMsg == null){ - return Status.JOB_SETUP_INPUT_FILE_ERROR.getName(); - } - String statusFileMsg = ws.createStatusFile(workspaceFolder, ws.getStatusFilePath(jobFolder), getHpcAddress()); - if(statusFileMsg == null){ - return null; - } - String jsonInputFileMsg = ws.createJSONInputFile(workspaceFolder, ws.getJSONInputFilePath(jobFolder), jsonString); - if(jsonInputFileMsg == null){ - return null; - } - String scriptFileMsg = ws.copyFile(slurmScript.getAbsolutePath(), jobFolder.getAbsolutePath().concat(File.separator).concat(slurmScript.getName())); - if(scriptFileMsg == null){ - return null; - } - String executableFileMsg = ws.copyFile(executable.getAbsolutePath(), jobFolder.getAbsolutePath().concat(File.separator).concat(executable.getName())); - if(executableFileMsg == null){ - return null; - } - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - } - - /** - * Shows the following statistics of Slurm jobs.
- * - Total number of jobs submitted - * - Total number of jobs currently running - * - Total number of jobs successfully completed - * - Total number of jobs terminated with an error - * - Total number of jobs not started yet - * - * @param input the JSON string specifying the return data format, e.g. JSON. - * @return the statistics in JSON format if requested. - */ - public String produceStatistics(String input) throws IOException, SlurmJobException{ - System.out.println("Received a request to send statistics.\n"); - LOGGER.info("Received a request to send statistics.\n"); - return ""; - } - - /** - * Shows the following statistics of Slurm jobs.
- * - Total number of jobs submitted - * - Total number of jobs currently running - * - Total number of jobs successfully completed - * - Total number of jobs terminated with an error - * - Total number of jobs not started yet - * - * @return the statistics in HTML format. - */ - public String showStatistics() throws IOException, SlurmJobException{ - System.out.println("Received a request to show statistics.\n"); - LOGGER.info("Received a request to show statistics.\n"); - return ""; - } - - /** - * Produces the statistics about Slurm jobs. - * - * @param input in json format containing the format in which the result
- * should be codified, e.g. json. An example input file will look like
- * as follows:{"format":"json"} - * @return a JSONObject containing statistics in the user requested format. - * @throws IOException - */ - public JSONObject getStatistics(String input) throws IOException{ - JSONObject obj = new JSONObject(input); - String formatAccepts = obj.optString("format"); // json - if(formatAccepts!=null && formatAccepts.toLowerCase().contains("json")){ - JobStatistics jobStatistics = new JobStatistics(workspaceDirectory); - obj = new JSONObject(); - obj.put("Number of jobs currently running", jobStatistics.getJobsRunning()); - obj.put("Number of jobs successfully completed", jobStatistics.getJobsCompleted()); - obj.put("Number of jobs completing", jobStatistics.getJobsCompleting()); - obj.put("Number of jobs failed", jobStatistics.getJobsFailed()); - obj.put("Number of jobs pending", jobStatistics.getJobsPending()); - obj.put("Number of jobs preempted", jobStatistics.getJobsPreempted()); - obj.put("Number of jobs suspended", jobStatistics.getJobsSuspended()); - obj.put("Number of jobs stopped", jobStatistics.getJobsStopped()); - obj.put("Number of jobs terminated with an error", jobStatistics.getJobsErrorTerminated()); - obj.put("Number of jobs to start", jobStatistics.getJobsNotStarted()); - obj.put("Total number of jobs submitted", jobStatistics.getJobsSubmitted()); - } - return obj; - } - - /** - * Produces the statistics about Slurm jobs. - * - * @return - * @throws IOException - */ - public String getStatistics() throws IOException{ - JobStatistics jobStatistics = new JobStatistics(workspaceDirectory); - String statistics = jobStatistics.getHTMLHeader(); - statistics = statistics + ""; - statistics = statistics.concat(jobStatistics.getBodydivStart()); - statistics = statistics + "
"; - String headerText = "Statistics about jobs submitted to this agent are shown in the table below.

"; - statistics = statistics.concat(jobStatistics.getStatisticsTableHeader(headerText, "Property", "Value", "50%")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs currently running", jobStatistics.getJobsRunning()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs successfully completed", jobStatistics.getJobsCompleted()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs completing", jobStatistics.getJobsCompleting()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs failed", jobStatistics.getJobsFailed()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs pending", jobStatistics.getJobsPending()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs preempted", jobStatistics.getJobsPreempted()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs suspended", jobStatistics.getJobsSuspended()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs stopped", jobStatistics.getJobsStopped()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs terminated with an error", jobStatistics.getJobsErrorTerminated()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Number of jobs to start", jobStatistics.getJobsNotStarted()+"")); - statistics = statistics.concat(jobStatistics.getStatisticsTableRow("Total number of jobs submitted", jobStatistics.getJobsSubmitted()+"")); - statistics = statistics + "

"+statisticsProperty+""+statisticsValue+"
"+property+""+value+"
"; - statistics = statistics + ""; - statistics = statistics + ""; - statistics = statistics + ""; - return statistics; - } - - /** - * Monitors the currently running Slurm jobs to allow new jobs to start.
- * In doing so, it checks if the number of running jobs is less than the
- * maximum number of jobs allowed to run at a time. - * - */ - public void monitorJobs() throws SlurmJobException{ - try { - if(!hostAvailabilityCheck(getHpcAddress(), 22)){ - System.out.println("The agent cannot connect to the HPC server with address " + getHpcAddress()); - session = null; - return; - } - scheduledIteration++; - if (session == null || scheduledIteration%10==0) { - if(session!=null && session.isConnected()){ - session.disconnect(); - } - System.out.println("Initialising a session."); - session = jsch.getSession(slurmJobProperty.getHpcServerLoginUserName(), getHpcAddress(), 22); - String pwd = slurmJobProperty.getHpcServerLoginUserPassword(); - session.setPassword(pwd); - String privateKeyFilename =slurmJobProperty.getHpcServerPrivateKey(); - - // Note that session.setConfig("PreferredAuthentications", ...) was removed because it will cause issues - // when this code is executed in a container for unknown reasons - try { - if(!privateKeyFilename.isEmpty()){ - jsch.addIdentity(privateKeyFilename); - }else{ - // Attempt to connect to a running instance of Pageant - Connector con = new PageantConnector(); - IdentityRepository irepo = new RemoteIdentityRepository(con); - jsch.setIdentityRepository(irepo); - } - } catch (AgentProxyException e) { - LOGGER.info("Failed to detect Pageant, will authenticate using password"); - } - - session.setConfig("StrictHostKeyChecking", "no"); - session.connect(); - scheduledIteration = 0; - } - if(workspaceDirectory.isDirectory()){ - File[] jobFolders = workspaceDirectory.listFiles(); - updateRunningJobSet(jobFolders, jobsRunning); - - for(File jobFolder: jobFolders){ - if(!Utils.isJobCompleted(jobFolder, slurmJobProperty)){ - if(Utils.isJobRunning(jobFolder)){ - if(updateRunningJobsStatus(jobFolder)){ - if(jobsRunning.contains(jobFolder.getName())){ - jobsRunning.remove(jobFolder.getName()); - } - } - } else if(Utils.isJobNotStarted(jobFolder) && !jobsRunning.contains(jobFolder.getName())){ - - if(jobsRunning.size() jobsRunning) throws IOException{ - for(File jobFolder: jobFolders){ - if(!(new File(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()))).exists()){ - LOGGER.info("SlurmJobAPI: job status file is not found, so the job folder with ID "+ jobFolder.getName()+" is being moved to the failed job folder."); - System.out.println("SlurmJobAPI: job status file is not found, so the job folder with ID "+ jobFolder.getName()+" is being moved to the failed job folder."); - try{ - Utils.moveToFailedJobsFolder(jobFolder, slurmJobProperty); - }catch(Exception e){ - LOGGER.info("SlurmJobAPI: failed to move the job folder with ID "+jobFolder.getName()+" to the failed job folder."); - System.out.println("SlurmJobAPI: failed to move the job folder with ID "+jobFolder.getName()+" to the failed job folder."); - } - continue; - } - try { - if (Utils.isJobRunning(jobFolder)) { - jobsRunning.add(jobFolder.getName()); - } - } catch (Exception e) { - LOGGER.info("SlurmJobAPI: failed to check the status of the job with ID "+jobFolder.getName()+ " while checking if it was running."); - System.out.println("SlurmJobAPI: failed to check the status of the job with ID "+jobFolder.getName()+ " while checking if it was running."); - } - } - } - - /** - * Starts running Slurm jobs which were set up before. - * - * @param jobFolder - * @throws SftpException - * @throws JSchException - * @throws IOException - * @throws UnknownHostException - * @throws InterruptedException - */ - private boolean runNotStartedJob(File jobFolder) throws SftpException, JSchException, IOException, UnknownHostException, InterruptedException{ - // A counter variable to count the number of mandatory files. - int countNumberOfFilesSetInProperties = 0; - - // Checks if the script file (including name and extension) for the current Slurm job is provided. - if(slurmJobProperty.getSlurmScriptFileName()==null || slurmJobProperty.getSlurmScriptFileName().isEmpty()){ - throw new IOException("SlurmJobAPI: Slurm script file name and extension are not provided."); - }else{ - countNumberOfFilesSetInProperties++; - } - // Checks if the input file name for the current Slurm job is provided. - if(slurmJobProperty.getInputFileName()==null || slurmJobProperty.getInputFileName().isEmpty()){ - throw new IOException("SlurmJobAPI: input file name is not provided."); - }else{ - countNumberOfFilesSetInProperties++; - } - // Checks if input file extension for the current Slurm job is provided. - if(slurmJobProperty.getInputFileExtension()==null || slurmJobProperty.getInputFileExtension().isEmpty()){ - throw new IOException("SlurmJobAPI: input file extension is not provided."); - } - // Checks if the JSON input file name for the current Slurm job is provided. - if(slurmJobProperty.getJsonInputFileName()==null || slurmJobProperty.getJsonInputFileName().isEmpty()){ - throw new IOException("SlurmJobAPI: JSON input file name is not provided."); - }else{ - countNumberOfFilesSetInProperties++; - } - // Checks if the JSON input file extension for the current Slurm job is provided. - if(slurmJobProperty.getJsonFileExtension()==null || slurmJobProperty.getJsonFileExtension().isEmpty()){ - throw new IOException("SlurmJobAPI: JSON file extension is not provided."); - } - // Checks if the executable file (including name and extension) for the current Slurm job is provided. - if(slurmJobProperty.getExecutableFile()!=null && !slurmJobProperty.getExecutableFile().isEmpty()){ - countNumberOfFilesSetInProperties++; - } - int countNumberOfFilesInJobFolder = 0; - // Checks the availability of the following four mandatory files in the current job folder: - // 1. Slurm script file (e.g., Slurm.sh) - // 2. Input file (e.g., input.zip or input.com) - // 3. JSON input file (e.g., input.json) - // 4. Status file (e.g., status.txt) - for(File file:jobFolder.listFiles()){ - if(file.getName().equalsIgnoreCase(slurmJobProperty.getSlurmScriptFileName())){ - countNumberOfFilesInJobFolder++; - } - if(file.getName().endsWith(slurmJobProperty.getInputFileExtension())){ - countNumberOfFilesInJobFolder++; - } - if(file.getName().equalsIgnoreCase(slurmJobProperty.getJsonInputFileName().concat(slurmJobProperty.getJsonFileExtension()))){ - countNumberOfFilesInJobFolder++; - } - if(file.getName().equalsIgnoreCase(Status.STATUS_FILE.getName())){ - countNumberOfFilesInJobFolder++; - } - if(slurmJobProperty.getExecutableFile()!=null && file.getName().equalsIgnoreCase(slurmJobProperty.getExecutableFile())){ - countNumberOfFilesInJobFolder++; - } - } - try{ - // If all files set through properties are not available in a job folder, it - // deletes the folder. - System.out.println("countNumberOfFilesInJobFolder:"+countNumberOfFilesInJobFolder); - System.out.println("countNumberOfFilesSetInProperties:"+countNumberOfFilesSetInProperties); - if(!(countNumberOfFilesSetInProperties>=3 && countNumberOfFilesSetInProperties+1==countNumberOfFilesInJobFolder)){ - LOGGER.info("SlurmJobAPI: all mandatory files are not found, so the job folder with ID "+ jobFolder.getName()+" is deleted."); - System.out.println("SlurmJobAPI: all mandatory files are not found, so the job folder with ID "+ jobFolder.getName()+" is deleted."); - Utils.moveToFailedJobsFolder(jobFolder, slurmJobProperty); - return false; - } - }catch(Exception e){ - LOGGER.info("SlurmJobAPI: all mandatory files are not found and an attempt to move the job folder with ID " - +jobFolder.getName()+" to the failed job folder is not successful."); - System.out.println("SlurmJobAPI: all mandatory files are not found and an attempt to move the job folder with ID " - +jobFolder.getName()+" to the failed job folder is not successful."); - } - - try{ - startJob(jobFolder.getName(), Arrays.asList(jobFolder.listFiles())); - }catch(Exception e){ - LOGGER.info("SlurmJobAPI: the Slurm Job with ID "+jobFolder.getName()+" could not be started.", e); - System.out.println("SlurmJobAPI: the Slurm Job with ID "+jobFolder.getName()+" could not be started."); - return false; - } - return true; - } - - /** - * Starts a Slurm job. - * - * @param job - * @param jobFiles - * @throws SftpException - * @throws JSchException - * @throws IOException - * @throws UnknownHostException - * @throws InterruptedException - */ - private void startJob(String job, List jobFiles) throws SftpException, JSchException, IOException, UnknownHostException, InterruptedException{ - // On HPC, the job folder and files have the address of machine where DFT Agent runs. - // Therefore, the HPC address is replaced with the address of Agent machine. - job = job.replace(getHpcAddress(), Utils.getMachineAddress()); - String jobFolderOnHPC = createJobFolder(job); - if(jobFolderOnHPC!=null){ - uploadFiles(jobFolderOnHPC, jobFiles); - } - } - - /** - * For running the current job, it uploads both the input file and
- * Slurm script to an HPC. - * - * @param jobFolderOnHPC - * @param jobFiles - * @throws SftpException - * @throws JSchException - * @throws IOException - * @throws UnknownHostException - * @throws InterruptedException - */ - private void uploadFiles(String jobFolderOnHPC, List jobFiles) throws SftpException, JSchException, IOException, UnknownHostException, InterruptedException{ - String replacedInputFileName = ""; - String statusFileAbsolutePath = ""; - String jobId = ""; - - for(File jobFile:jobFiles){ - - if(jobFile.getAbsolutePath().endsWith(slurmJobProperty.getInputFileExtension())){ - replacedInputFileName = getInputFileNameReplaced(jobFile.getAbsolutePath()); - String inputFileNameOnHPC = jobFolderOnHPC.concat("/").concat(replacedInputFileName); - uploadFile(jobFile.getAbsolutePath(), inputFileNameOnHPC); - replaceFileContent(jobFolderOnHPC, inputFileNameOnHPC); - - }else if(!jobFile.getAbsolutePath().endsWith(Property.STATUS_FILE_NAME.getPropertyName())){ - if(jobFile.getAbsolutePath().endsWith(".sh")){ - Utils.translateLineEndingIntoUnix(new File(jobFile.getAbsolutePath())); - } - uploadFile(jobFile.getAbsolutePath(), jobFolderOnHPC); - } - - if(jobFile.getAbsolutePath().endsWith(Property.STATUS_FILE_NAME.getPropertyName())){ - statusFileAbsolutePath = jobFile.getAbsolutePath(); - } - } - - LOGGER.info("Attemping to run job folder on HPC at: " + jobFolderOnHPC); - jobId = runSlurmJob(jobFolderOnHPC, replacedInputFileName); - if(!jobId.isEmpty()){ - Utils.addJobId(statusFileAbsolutePath, jobId); - } - } - - /** - * Produces the command to go to the current job directory and to run
- * the job. - * - * @param jobFolderOnHPC - * @param inputFile - * @return - * @throws JSchException - * @throws IOException - */ - private String runSlurmJob(String jobFolderOnHPC, String inputFile) throws JSchException, IOException{ - inputFile = inputFile.replace(slurmJobProperty.getInputFileExtension(), ""); - String command = "cd " - .concat(jobFolderOnHPC) - .concat(" && ") - .concat("sbatch --job-name=").concat(inputFile) - .concat(" ") - .concat(Property.SLURM_SCRIPT_FILE_NAME.getPropertyName()); - - LOGGER.info("Using command: " + command); - return runSlurmJob(command); - } - - /** - * Runs a Slurm job. - * - * @param command - * @return - * @throws JSchException - * @throws IOException - */ - private String runSlurmJob(String command) throws JSchException, IOException{ - ArrayList outputs = executeCommand(command); - if (outputs == null) { - return null; - } - String jobId = getJobId(outputs); - System.out.println("Job id:" + jobId); - return jobId; - } - - /** - * Modifies the names of the check point file and log file provided in
- * the input file after copying this into the HPC cluster. - * - * @param jobFolderOnHPC - * @param replacedJobFileName - * @throws JSchException - * @throws IOException - * @throws UnknownHostException - */ - private void replaceFileContent(String jobFolderOnHPC, String replacedJobFileName) throws JSchException, IOException, UnknownHostException{ - String command = "cd ".concat(jobFolderOnHPC).concat(" && ") - .concat("sed -i 's/").concat(getHpcAddress()) - .concat("/").concat(Utils.getMachineAddress()).concat("/g' ").concat(replacedJobFileName); - executeCommand(command); - } - - /** - * Modifies the name of the input file while copying this into the HPC cluster. - * - * @param inputFile - * @return - * @throws UnknownHostException - */ - private String getInputFileNameReplaced(String inputFile) throws UnknownHostException{ - String tokens[]; - if(inputFile.contains("/")){ - tokens = inputFile.split("/"); - }else{ - tokens = inputFile.split("\\\\"); - } - return tokens[tokens.length-1].replace(getHpcAddress(), Utils.getMachineAddress()); - } - - private String createJobFolder(String job) throws JSchException, IOException{ - // Creates the "mkdir" (make directory) command to create the workspace/jobspace directory. - String command = "mkdir /home/".concat(slurmJobProperty.getHpcServerLoginUserName()).concat("/").concat(workspaceDirectory.getName()); - // Executes the command to create the workspace/jobspace directory. - executeCommand(command); - // Creates the command to create the job directory. - command = "mkdir /home/".concat(slurmJobProperty.getHpcServerLoginUserName()).concat("/").concat(workspaceDirectory.getName()).concat("/").concat(job); - // Executes the command for creating the job directory. - executeCommand(command); - return "/home/".concat(slurmJobProperty.getHpcServerLoginUserName()).concat("/").concat(workspaceDirectory.getName()).concat("/").concat(job); - } - - /** - * Updates the status of a currently running job. - * - * @param jobFolder - * @return - * @throws JSchException - * @throws SftpException - * @throws IOException - * @throws InterruptedException - */ - private boolean updateRunningJobsStatus(File jobFolder) - throws JSchException, SftpException, IOException, InterruptedException { - boolean status = false; - try{ - File statusFile = Utils.getStatusFile(jobFolder); - status = updateRunningJobsStatus(jobFolder.getName(), statusFile); - }catch(Exception e){ - LOGGER.info("SlurmJobAPI: failed to update the status of the job with ID "+jobFolder.getName()+" while checking if it was still running."); - } - return status; - } - - /** - * Updates the latest status of the running jobs. - * - * @param runningJob - * @param statusFile - * @return - * @throws JSchException - * @throws SftpException - * @throws IOException - * @throws InterruptedException - */ - private boolean updateRunningJobsStatus(String runningJob, File statusFile) throws JSchException, SftpException, IOException, InterruptedException{ - if(statusFile!=null){ - if(!isJobRunning(statusFile)){ - if(slurmJobProperty.getOutputFileExtension().trim().toLowerCase().equals(".log")){ - if(outputFileExist(Utils.getLogFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress()))){ - try{ - downloadFile(Utils.getLogFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress()), Utils.getJobOutputFilePathOnAgentPC(runningJob, workspaceDirectory, runningJob, Status.EXTENSION_LOG_FILE.getName())); - updateStatusForErrorTermination(statusFile, Utils.getJobOutputFilePathOnAgentPC(runningJob, workspaceDirectory, runningJob, Status.EXTENSION_LOG_FILE.getName())); - }catch(Exception e){ - Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName()); - } - }else{ - Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName()); - } - }else{ - if(outputFileExist(Utils.getOutputFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress(), slurmJobProperty.getOutputFileName().concat(slurmJobProperty.getOutputFileExtension())))){ - try{ - downloadFile(Utils.getOutputFilePathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress(), slurmJobProperty.getOutputFileName().concat(slurmJobProperty.getOutputFileExtension())), Utils.getJobOutputFilePathOnAgentPC(runningJob, workspaceDirectory, slurmJobProperty.getOutputFileName(), slurmJobProperty.getOutputFileExtension())); - }catch(Exception e){ - Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName()); - } - }else{ - Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName()); - } - } - deleteJobOnHPC(Utils.getJobFolderPathOnHPC(runningJob, slurmJobProperty.getHpcServerLoginUserName(), workspaceDirectory, getHpcAddress())); - return true; - } - } - return false; - } - - /** - * Checks if the output file exists. If it does, it returns true, it returns false otherwise. - * - * @param fileAbsoultePath - * @return - * @throws JSchException - * @throws IOException - */ - private boolean outputFileExist(String fileAbsoultePath) throws JSchException, IOException{ - String command = "[ -f "+fileAbsoultePath+" ] && echo "+Status.JOB_OUTPUT_FILE_EXIST_MESSAGE.getName(); - ArrayList outputs = executeCommand(command); - if(outputs!=null && outputs.size()>0){ - if(outputs.contains(Status.JOB_OUTPUT_FILE_EXIST_MESSAGE.getName())){ - return true; - } - } - return false; - } - - /** - * Extracts from the log file if the current job terminated normally or
- * with error. In the case of error termination, updates the status of
- * the log file accordingly. - * - * @param statusFile - * @param jobFolderPathOnAgentPC - */ - private void updateStatusForErrorTermination(File statusFile, String jobFolderPathOnAgentPC) throws IOException{ - if(Utils.isErrorTermination(jobFolderPathOnAgentPC)){ - Utils.modifyStatus(statusFile.getAbsolutePath(), Status.JOB_LOG_MSG_ERROR_TERMINATION.getName()); - } - } - - /** - * Checks if a job is running using the job id. - * - * @param statusFile - * @return - * @throws JSchException - * @throws IOException - * @throws InterruptedException - */ - private boolean isJobRunning(File statusFile) throws JSchException, IOException, InterruptedException{ - String jobId = Utils.getJobId(statusFile.getAbsolutePath()); - if(jobId==null){ - return false; - } else{ - boolean isJobRunning = isJobRunning(jobId , statusFile); - if(isJobRunning){ - return true; - }else{ - Utils.modifyStatus(statusFile.getAbsolutePath(), Status.STATUS_JOB_COMPLETED.getName()); - return false; - } - } - } - - /** - * Checks the status of job on the HPC where it was submitted. - * - * @param jobId - * @param statusFile - * @return - * @throws IOException - * @throws InterruptedException - */ - private boolean isJobRunning(String jobId, File statusFile) throws JSchException, IOException, InterruptedException{ - String command = "squeue -j " + jobId; - ArrayList outputs = executeCommand(command); - boolean jobRunning = isJobRunning(outputs, statusFile); - return jobRunning; - } - - /** - * Analyses the outputs following the execution of the
- * job status check command to understand if the job
- * is still running or terminated. - * - * @param outputs - * @return - */ - private boolean isJobRunning(ArrayList outputs, File statusFile) throws IOException{ - if(outputs!=null && outputs.size()<=1){ - return false; - } - if(outputs!=null){ - String[] headers = outputs.get(0).split("\\s+"); - String[] values = outputs.get(1).split("\\s+"); - for(int i=0; ii){ -// updateStatus(values[i], statusFile); - } - } - } - } - return true; - } - - /** - * Extracts the job id from the - * - * @param outputs - * @return - */ - private String getJobId(ArrayList outputs){ - for(String output: outputs){ - if(output.contains("Submitted batch job")){ - String tokens[] = output.split(" "); - if(tokens.length>=4){ - return tokens[3].trim(); - } - } - } - return null; - } - - /** - * Deletes the job folder including all files and folders belonging to it. - * - * @param jobFolder the absolute path to the job folder. - * @throws JSchException - * @throws IOException - * @throws InterruptedException - */ - private void deleteJobOnHPC(String jobFolder) throws JSchException, IOException, InterruptedException{ - String command = "rm -r " + jobFolder; - executeCommand(command); - } - - /** - * Reads the output of the most recently executed command. - * - * @param br - * @return - * @throws IOException - */ - public ArrayList readCommandOutput(BufferedReader br) throws IOException{ - if(br == null) - throw new IOException("The reader is not initialised."); - String line; - ArrayList outputs = new ArrayList<>(); - while((line=br.readLine())!=null){ - outputs.add(line); - } - br.close(); - return outputs; - } - - /** - * Displays every item in a list of string array. - * - * @param list - */ - public void displayArray(ArrayList list){ - for(String item:list){ - System.out.println(item); - } - } - - /** - * Uploads a file from the source folder belonging to any machine to
- * the destination folder on the same machine or any other machine. - * - * @param src - * @param dest - * @throws JSchException - * @throws SftpException - */ - public void uploadFile(String src, String dest) throws JSchException, SftpException{ - System.out.println("Establishing a channel to transfer "+src+" to "+dest); - ChannelSftp sftpChannel = (ChannelSftp) session.openChannel("sftp"); - sftpChannel.connect(); - - sftpChannel.put(src, dest); - sftpChannel.disconnect(); - System.out.println("Closing the channel."); - } - - /** - * Downloads the log file. - * - * @param src the file to be downloaded - * @param dest the path under which the source file will be downloaded
- * or the absolute path containing the path and the file name. - * @throws JSchException - * @throws SftpException - */ - public void downloadFile(String src, String dest) throws JSchException, SftpException{ - System.out.println("Establishing a channel to transfer "+src+" to "+dest); - ChannelSftp sftpChannel = (ChannelSftp) session.openChannel("sftp"); - sftpChannel.connect(); - sftpChannel.get(src, dest); - sftpChannel.disconnect(); - System.out.println("Closing the channel."); - } - - /** - * Deletes a folder or file name from an HPC, if the complete path is provided.
- * For example, to delete a folder called "test", user needs to provide "rds/.../test" - * - * @param folderOrFileName - * @throws JSchException - * @throws SftpException - */ - public void deleteFolderOrFile(String folderOrFileName) throws JSchException, SftpException, IOException{ - executeCommand("rm -r "+folderOrFileName); - } - - /** - * Executes any command that can be run on the Linux console. - * - * @param Command - * @return - * @throws JSchException - * @throws IOException - */ - public ArrayList executeCommand(String Command) throws JSchException, IOException{ - ArrayList outputs = null; - System.out.println("Establishing a channel to perform the following command:" + Command); - Channel execChannel = session.openChannel("exec"); - ((ChannelExec) execChannel).setCommand(Command); - execChannel.setInputStream(null); - ((ChannelExec) execChannel).setErrStream(System.err); - BufferedReader stdInput = new BufferedReader(new InputStreamReader(execChannel.getInputStream())); - execChannel.connect(); - outputs = readCommandOutput(stdInput); - execChannel.disconnect(); - System.out.println("Closing the channel."); - return outputs; - } - - /** - * Indicates if a server is online. - * - * @param server refers to the server address - * @param port refers to the port number - * @return - */ - public boolean hostAvailabilityCheck(String server, int port) throws IOException { - boolean available = true; -// try (final Socket dummy = new Socket(server, port)){ -// } catch (UnknownHostException | IllegalArgumentException e) { -// available = false; -// } - return available; - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessing.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessing.java deleted file mode 100644 index 1f97fa416..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessing.java +++ /dev/null @@ -1,77 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.SftpException; - -import uk.ac.cam.cares.jps.base.util.FileUtil; - - -public class PostProcessing { - /** - * Updates the output status of a completed job. - * - * @param jobFolder - * @return - * @throws JSchException - * @throws SftpException - * @throws IOException - * @throws InterruptedException - */ - public static boolean updateJobOutputStatus(File jobFolder) - throws IOException { - File statusFile = Utils.getStatusFile(jobFolder); - return updateJobOutputStatus(statusFile, Status.OUTPUT_PROCESSED.getName()); - } - - /** - * Updates the latest status of the running jobs. - * - * @param runningJob - * @param statusFile - * @return - * @throws JSchException - * @throws SftpException - * @throws IOException - * @throws InterruptedException - */ - private static boolean updateJobOutputStatus(File statusFile, String status) throws IOException{ - if(statusFile!=null){ - modifyOutputStatus(statusFile.getAbsolutePath(), Status.OUTPUT_PROCESSED.getName()); - return true; - } - return false; - } - - /** - * Modifies the output to processed in the status file. - * - * @param filePath the path to the status file. - * @param status can be empty ("") or "processed". - * @throws IOException - */ - public static void modifyOutputStatus(String filePath, String status) throws IOException{ - List fileContent = new ArrayList<>(); - BufferedReader br = FileUtil.openSourceFile(filePath); - String line; - while((line=br.readLine())!=null){ - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_OUTPUT.getName())) { - line = Status.ATTRIBUTE_JOB_OUTPUT.getName().concat(" ").concat(status); - } - fileContent.add(line); - } - br.close(); - BufferedWriter bw = FileUtil.openBufferedWriter(filePath); - for(String lineContent:fileContent){ - bw.write(lineContent.concat("\n")); - } - bw.flush(); - bw.close(); - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Property.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Property.java deleted file mode 100644 index 1ee69f85c..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Property.java +++ /dev/null @@ -1,38 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -/** - * This enumerated list defines the name of important properties of
- * Slurm jobs. Some example properties are:
- * - the name of the agent class
- * - the name of the workspace folder on the machine where the agent runs
- * - the name of the workspace folder on HPC where DFT calculations run
- * - * @author Feroz Farazi(msff2@cam.ac.uk) - * - */ -public enum Property { - - JOB_WORKSPACE_PARENT_DIR(System.getProperty("user.home")), - CHK_POINT_FILE_EXTENSION(".chk"), - STATUS_FILE_NAME("status.txt"), - JSON_INPUT_FILE_NAME("input.json"), - SLURM_SCRIPT_FILE_NAME("Slurm.sh"); - - private String propertyName; - private int value; - private Property(String propertyName){ - this.propertyName = propertyName; - } - - public String getPropertyName(){ - return propertyName; - } - - private Property(final int newValue){ - value = newValue; - } - - public int getValue(){ - return value; - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJob.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJob.java deleted file mode 100644 index 75d29c2f0..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJob.java +++ /dev/null @@ -1,23 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -public class SlurmJob { - private String jobFolderName; - - public String getJobFolderName() { - return jobFolderName; - } - - public void setJobFolderName(String jobFolderName) { - this.jobFolderName = jobFolderName; - } - /** - * To create a new job folder name, this method is used. - * - * @param hpcAddress - * @return - */ - public String getNewJobFolderName(String hpcAddress){ - setJobFolderName(hpcAddress.concat("_").concat("" + Utils.getTimeStamp())); - return getJobFolderName(); - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobException.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobException.java deleted file mode 100644 index 82ae1fc9e..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobException.java +++ /dev/null @@ -1,32 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -/** - * The exception class to catch all possible exceptions within the
- * Slurm Job API. - * - * @author msff2 - * - */ -public class SlurmJobException extends Exception{ - public SlurmJobException() { - super(); - } - - public SlurmJobException(String message) { - super(message); - } - - public SlurmJobException(String message, Throwable cause) { - super(message, cause); - } - - public SlurmJobException(Throwable cause) { - super(cause); - } - - protected SlurmJobException(String message, Throwable cause, - boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Status.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Status.java deleted file mode 100644 index a06eaed83..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Status.java +++ /dev/null @@ -1,57 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -/** - * Starting from the submission to the completion a Slurm job goes through
- * different statuses, e.g. pending, running and completed. This java class
- * contains all possible statuses and corresponding messages as constants. - * - * @author msff2 - * - */ -public enum Status { - STATUS_FILE("status.txt"), - ATTRIBUTE_JOB_STATUS("JobStatus:"), - STATUS_JOB_NOT_STARTED("not started"), - STATUS_JOB_COMPLETED("completed"), - STATUS_JOB_COMPLETING("completing"), - STATUS_JOB_FAILED("failed"), - STATUS_JOB_PENDING("pending"), - STATUS_JOB_PREEMPTED("preempted"), - STATUS_JOB_RUNNING("running"), - STATUS_JOB_SUSPENDED("suspended"), - STATUS_JOB_STOPPED("stopped"), - STATUS_JOB_ERROR_TERMINATED("error termination"), - ATTRIBUTE_JOB_ID("JobId:"), - ATTRIBUTE_AGENT_ID("AgentId:"), - ATTRIBUTE_HPC_ADDRESS("HPCAddress:"), - ATTRIBUTE_JOB_OUTPUT("JobOutput:"), - OUTPUT_PROCESSED("processed"), - EXTENSION_SLURM_FILE(".sh"), - EXTENSION_LOG_FILE(".log"), - EXTENSION_JSON_FILE(".json"), - JOB_SETUP_SUCCESS_MSG("The job request has been processed successfully."), - JOB_SETUP_JSON_ERROR("The JSON request is ill formatted."), - JOB_SETUP_SLURM_SCRIPT_ERROR("The Slurm script could not be downloaded."), - JOB_SETUP_INPUT_FILE_ERROR("The input file for the job could not be created."), - JOB_SETUP_ERROR("The requested job could not be set up, therefore, resubmit the job."), - JOB_LOG_MSG_ERROR_TERMINATION("error termination"), - JOB_LOG_MSG_NORMAL_TERMINATION("normal termination"), - JOB_OUTPUT_FILE_EXIST_MESSAGE("The file exist."), - JOB_SQUEUE_STATUS_COMPLETED("CD"), - JOB_SQUEUE_STATUS_COMPLETING("CG"), - JOB_SQUEUE_STATUS_FAILED("F"), - JOB_SQUEUE_STATUS_PENDING("PD"), - JOB_SQUEUE_STATUS_PREEMPTED("PR"), - JOB_SQUEUE_STATUS_RUNNING("R"), - JOB_SQUEUE_STATUS_SUSPENDED("S"), - JOB_SQUEUE_STATUS_STOPPED("ST"); - - private String property; - private Status(String property){ - this.property = property; - } - - public String getName(){ - return property; - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Utils.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Utils.java deleted file mode 100644 index 02f83ecb6..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Utils.java +++ /dev/null @@ -1,626 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import uk.ac.cam.cares.jps.base.slurm.job.configuration.SlurmJobProperty; - -public class Utils{ - private static Logger LOGGER = LogManager.getLogger(Utils.class); - public static long previousTimeStamp; - - /** - * Generate unique time stamp to be used in naming Slurm jobs. - * - * @return - */ - public static long getTimeStamp(){ - long currentTimeStamp = System.nanoTime(); - while(!(currentTimeStamp > previousTimeStamp)){ - currentTimeStamp = System.nanoTime(); - } - previousTimeStamp = currentTimeStamp; - return currentTimeStamp; - } - - /** - * Creates an instance of the BufferedWriter class. - * - * @param filePathPlusName the path plus name of the file being written - * @return - * @throws IOException - */ - public static BufferedWriter openBufferedWriter(String filePathPlusName) throws IOException{ - return new BufferedWriter(new FileWriter(filePathPlusName)); - } - - /** - * Creates and returns an instance of the BufferedReader class. - * - * @param filePathPlusName - * the path plus name of the file being read - * @return - * @throws IOException - */ - public static BufferedReader openSourceFile(String filePathPlusName) - throws IOException { - return new BufferedReader(new InputStreamReader(new FileInputStream( - filePathPlusName), "UTF-8")); - } - - /** - * Check if a job is completed. - * - * @param jobFolder - * @return - * @throws IOException - */ - public static boolean isJobCompleted(File jobFolder) throws IOException{ - return isJobFinished(jobFolder, jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())); - } - - /** - * Check if a job is completed. - * - * @param jobFolder - * @param slurmJobProperty - * @return - * @throws IOException - */ - public static boolean isJobCompleted(File jobFolder, SlurmJobProperty slurmJobProperty) throws IOException{ - boolean status = false; - try { - status = isJobFinished(jobFolder, jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName()), slurmJobProperty); - } catch (Exception e) { - LOGGER.info("SlurmJobAPI: failed to check the status of the job with ID " + jobFolder.getName()); - } - return status; - } - - /** - * Check if a job is completed with erroneous or incomplete output. - * - * @param jobFolder - * @return - * @throws IOException - */ - public static boolean isJobErroneouslyCompleted(File jobFolder) throws IOException{ - return isJobErroneouslyCompleted(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())); - } - - /** - * Checks if a job is still running. - * - * @param jobFolder - * @return - * @throws IOException - */ - public static boolean isJobRunning(File jobFolder) throws IOException{ - boolean status = false; - try{ - status = isJobRunning(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())); - }catch(Exception e){ - LOGGER.info("SlurmJobAPI: failed to check the status of the job with ID "+jobFolder.getName()); - } - return status; - } - - /** - * Checks if a job is not started yet. - * - * @param jobFolder - * @return - * @throws IOException - */ - public static boolean isJobNotStarted(File jobFolder) throws IOException{ - return isJobNotStarted(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())); - } - - /** - * Check the status if a job finished. - * - * @param jobFolder the folder that contains the job. - * @param statusFilePath the file that contains the status of the job. - * @return - * @throws IOException - */ - public static boolean isJobFinished(File jobFolder, String statusFilePath) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){ - if(line.contains(Status.STATUS_JOB_COMPLETED.getName())){ - statusFile.close(); - return true; - } - if(line.contains(Status.STATUS_JOB_ERROR_TERMINATED.getName())){ - statusFile.close(); - return true; - } - } - } - statusFile.close(); - return false; - } - - - /** - * Check the status if a job finished. - * - * @param jobFolder the folder that contains the job. - * @param statusFilePath the file that contains the status of the job. - * @param slurmJobProperty the variable that allows to read different
- * properties including agentClass, completed jobs folder prefix, etc. - * - * @return - * @throws IOException - */ - public static boolean isJobFinished(File jobFolder, String statusFilePath, SlurmJobProperty slurmJobProperty) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){ - if(line.contains(Status.STATUS_JOB_COMPLETED.getName())){ - statusFile.close(); - if(isJobPostProcessed(jobFolder, statusFilePath)){ - moveToCompletedJobsFolder(jobFolder, slurmJobProperty); - } - return true; - } - if(line.contains(Status.STATUS_JOB_ERROR_TERMINATED.getName())){ - statusFile.close(); - moveToFailedJobsFolder(jobFolder, slurmJobProperty); - return true; - } - } - } - statusFile.close(); - return false; - } - - /** - * Check the status if a job post-processed. - * - * @param jobFolder the folder that contains the job. - * @param statusFilePath the file that contains the status of the job. - * @return - * @throws IOException - */ - public static boolean isJobPostProcessed(File jobFolder, String statusFilePath) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_OUTPUT.getName())){ - if(line.contains(Status.OUTPUT_PROCESSED.getName())){ - statusFile.close(); - return true; - } - } - } - statusFile.close(); - return false; - } - - /** - * Check the status if a job produced erroneous or incomplete output. - * - * @param statusFilePath - * @return - * @throws IOException - */ - public static boolean isJobErroneouslyCompleted(String statusFilePath) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){ - if(line.contains(Status.STATUS_JOB_ERROR_TERMINATED.getName())){ - statusFile.close(); - return true; - } - } - } - statusFile.close(); - return false; - } - - public static boolean isJobOutputProcessed(File jobFolder) throws IOException{ - return isJobOutputProcessed(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())); - } - - /** - * Check the status if a job is currently running. - * - * @param statusFilePath the absolute path to the status file. - * @return - * @throws IOException - */ - public static boolean isJobOutputProcessed(String statusFilePath) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_OUTPUT.getName())){ - if(line.contains(Status.OUTPUT_PROCESSED.getName())){ - statusFile.close(); - return true; - } - } - } - statusFile.close(); - return false; - } - - /** - * Check the status if a job is currently running. - * - * @param statusFilePath the absolute path to the status file. - * @return - * @throws IOException - */ - public static boolean isJobRunning(String statusFilePath) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){ - if(line.contains(Status.STATUS_JOB_RUNNING.getName()) - || line.contains(Status.STATUS_JOB_COMPLETING.getName()) - || line.contains(Status.STATUS_JOB_PENDING.getName())){ - statusFile.close(); - return true; - } - } - } - statusFile.close(); - return false; - } - - protected static boolean isStatusFileOpen = false; - - /** - * Check the status if a job is not started yet. - * - * @param statusFilePath the absolute path to the status file. - * @return - * @throws IOException - */ - public static boolean isJobNotStarted(String statusFilePath) throws IOException{ - if(isStatusFileOpen){ - return false; - } - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - isStatusFileOpen = true; - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())){ - if(line.contains(Status.STATUS_JOB_NOT_STARTED.getName())){ - statusFile.close(); - isStatusFileOpen = false; - return true; - } - } - } - statusFile.close(); - isStatusFileOpen = false; - return false; - } - - /** - * Reads the job id from the status file. - * - * @param statusFilePath the absolute path to the status file. - * @return - * @throws IOException - */ - public static String getJobId(String statusFilePath) throws IOException{ - BufferedReader statusFile = Utils.openSourceFile(statusFilePath); - String line; - while((line=statusFile.readLine())!=null){ - if(line.trim().toLowerCase().startsWith(Status.ATTRIBUTE_JOB_ID.getName().toLowerCase())){ - String tokens[] = line.trim().split(":"); - if(tokens.length>=2 && tokens[1].trim().length()>0){ - statusFile.close(); - return tokens[1].trim(); - } - } - } - statusFile.close(); - return null; - } - /** - * Adds the job id to the status file. - * - * @param filePath the path to the status file. - * @param jobId the job id generated following the sbatch submission. - * @throws IOException - */ - public static void addJobId(String filePath, String jobId) throws IOException{ - List fileContent = new ArrayList<>(); - BufferedReader br = openSourceFile(filePath); - String line; - while((line=br.readLine())!=null){ - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) { - line = Status.ATTRIBUTE_JOB_STATUS.getName().concat(" ").concat(Status.STATUS_JOB_RUNNING.getName()); - } - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName())) { - line = Status.ATTRIBUTE_JOB_ID.getName().concat(" ").concat(jobId); - } - fileContent.add(line); - } - br.close(); - BufferedWriter bw = openBufferedWriter(filePath); - for(String lineContent:fileContent){ - bw.write(lineContent.concat("\n")); - } - bw.flush(); - bw.close(); - } - - /** - * Modifies the status of job in the status file. - * - * @param filePath the path to the status file. - * @param status can be "running" or "completed". - * @throws IOException - */ - public static void modifyStatus(String filePath, String status) throws IOException{ - List fileContent = new ArrayList<>(); - BufferedReader br = openSourceFile(filePath); - String line; - while((line=br.readLine())!=null){ - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) { - line = Status.ATTRIBUTE_JOB_STATUS.getName().concat(" ").concat(status); - } - fileContent.add(line); - } - br.close(); - BufferedWriter bw = openBufferedWriter(filePath); - for(String lineContent:fileContent){ - bw.write(lineContent.concat("\n")); - } - bw.flush(); - bw.close(); - } - - /** - * Returns the status file if the job folder is provided. - * - * @param jobFolder path to the job folder. - * @return - */ - public static File getStatusFile(File jobFolder){ - if(jobFolder.isDirectory()){ - if((new File(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())).isFile())){ - return new File(jobFolder.getAbsolutePath().concat(File.separator).concat(Status.STATUS_FILE.getName())); - } - } - return null; - } - - /** - * Returns the log file path (absolute path) on a HPC server where the
- * Slurm job is running. - * - * @param runningJob - * @param userName - * @param taskSpace - * @param hpcAddress - * @return - * @throws UnknownHostException - */ - public static String getLogFilePathOnHPC(String runningJob, String userName, File taskSpace, String hpcAddress) throws UnknownHostException{ - String jobFolderOnHPC = runningJob.replace(hpcAddress, getMachineAddress()); - String logFilePath = getJobFolderPathOnHPC(runningJob, userName, taskSpace, hpcAddress).concat("/").concat(jobFolderOnHPC) - .concat(Status.EXTENSION_LOG_FILE.getName()); - return logFilePath; - } - - /** - * Returns the output file path (absolute path) on a HPC server where the
- * Slurm job is running. - * - * @param runningJob - * @param userName - * @param taskSpace - * @param hpcAddress - * @param outputFileNameWithExtension - * @return - * @throws UnknownHostException - */ - public static String getOutputFilePathOnHPC(String runningJob, String userName, File taskSpace, String hpcAddress, String outputFileNameWithExtension) throws UnknownHostException{ - String outputFilePath = getJobFolderPathOnHPC(runningJob, userName, taskSpace, hpcAddress).concat("/").concat(outputFileNameWithExtension); - return outputFilePath; - } - - /** - * Returns the job folder path (absolute path) on a HPC where the job is running. - * - * @param runningJob - * @param userName - * @param taskSpace - * @param hpcAddress - * @return - * @throws UnknownHostException - */ - public static String getJobFolderPathOnHPC(String runningJob, String userName, File taskSpace, String hpcAddress) throws UnknownHostException{ - String jobFolderOnHPC = runningJob.replace(hpcAddress, getMachineAddress()); - String jobFolderPath = "/home/".concat(userName).concat("/") - .concat(taskSpace.getName()).concat("/") - .concat(jobFolderOnHPC); - return jobFolderPath; - } - - /** - * Returns the absolute path to the output file on the machine where the Agent is running. - * - * @param runningJob - * @param taskSpace - * @param outputFileName - * @param outputFileExtension - * @return - */ - public static String getJobOutputFilePathOnAgentPC(String runningJob, File taskSpace, String outputFileName, String outputFileExtension){ - return taskSpace.getAbsolutePath().concat(File.separator).concat(runningJob).concat(File.separator) - .concat(outputFileName).concat(outputFileExtension); - } - - /** - * Checks the log file, in the case of the Gaussian software, if it is an
- * error termination. - * - * @param logFileOnAgentPC - * @return - * @throws IOException - */ - public static boolean isErrorTermination(String logFileOnAgentPC)throws IOException{ - BufferedReader logFile = openSourceFile(logFileOnAgentPC); - String line; - while((line=logFile.readLine())!=null){ - if(line.trim().toLowerCase().startsWith(Status.JOB_LOG_MSG_ERROR_TERMINATION.getName().toLowerCase())){ - logFile.close(); - return true; - } - } - logFile.close(); - return false; - } - - /** - * Reads the address of the machine where an instance of the agent
- * running Slurm jobs is hosted. - * - * @return - * @throws UnknownHostException - */ - public static String getMachineAddress() throws UnknownHostException{ - try { - InetAddress address; - address = InetAddress.getLocalHost(); - return address.toString().replace("/", "_"); - } catch (UnknownHostException e){ - e.printStackTrace(); - LOGGER.error("SlurmJobAPI: The host address is unknown as InetAddress.getLocalHost() threw an exception."); - } - return null; - } - - /** - * Windows uses both carriage return (\r) and line feed (\n) as a line
- * ending (\r\n). Unix uses line feed (\n) as a line ending.
- * This method translates all line endings codified with "\r\n" in a
- * file into "\n". - * - * @param file - */ - public static void translateLineEndingIntoUnix(File file) throws IOException{ - File recreatedFile = new File(System.getProperty("user.home").replace("\\", "/").concat("/").concat(file.getName())); - copyModifiedContentForUnix(file, recreatedFile); - copyModifiedContentForUnix(recreatedFile, file); - } - - /** - * Copies file from the source path to the destination path with the
- * line ending modified to format in Unix. - * - * @param source - * @param destination - * @throws IOException - */ - private static void copyModifiedContentForUnix(File source, File destination) throws IOException{ - BufferedReader receivedFile = openSourceFile(source.getAbsolutePath()); - BufferedWriter recreatedFile = openBufferedWriter(destination.getAbsolutePath()); - String line; - while((line=receivedFile.readLine())!=null){ - recreatedFile.write(line.concat("\n")); - } - recreatedFile.close(); - receivedFile.close(); - } - - /** - * Moves the provided job folder to the folder that contains completed
- * and post-processed jobs of the agent. - * - * @param jobFolder the folder that contains a job - * @param slurmJobProperty - */ - public static void moveToCompletedJobsFolder(File jobFolder, SlurmJobProperty slurmJobProperty) { - try { - File destDir = getCompletedJobsDirectory(jobFolder, slurmJobProperty); - if(destDir!=null){ - FileUtils.copyDirectory(jobFolder, destDir); - FileUtils.deleteDirectory(jobFolder); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * Returns the folder where completed jobs are saved. - * - * @param jobFolder - * @param slurmJobProperty - * @return - * @throws IOException - */ - public static File getCompletedJobsDirectory(File jobFolder, SlurmJobProperty slurmJobProperty) throws IOException{ - File workspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), slurmJobProperty.getAgentClass()); - String completedJobsDirectory = Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator).concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(workspace.getName()).concat(File.separator).concat(jobFolder.getName()); - File jobFolderInCompletedJobs = new File(completedJobsDirectory); - jobFolderInCompletedJobs.mkdirs(); - if(jobFolderInCompletedJobs.exists()){ - return jobFolderInCompletedJobs; - } - return null; - } - - /** - * Moves the provided job folder to the folder that contains failed
- * jobs of the agent. - * - * @param jobFolder the folder that contains a job - * @param slurmJobProperty - */ - public static void moveToFailedJobsFolder(File jobFolder, SlurmJobProperty slurmJobProperty) { - try { - File destDir = getFailedJobsDirectory(jobFolder, slurmJobProperty); - if(destDir!=null){ - FileUtils.copyDirectory(jobFolder, destDir); - FileUtils.deleteDirectory(jobFolder); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * Returns the folder where failed jobs are saved. - * - * @param jobFolder - * @param slurmJobProperty - * @return - * @throws IOException - */ - public static File getFailedJobsDirectory(File jobFolder, SlurmJobProperty slurmJobProperty) throws IOException{ - File workspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), slurmJobProperty.getAgentClass()); - String failedJobsDirectory = Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator).concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(workspace.getName()).concat(File.separator).concat(jobFolder.getName()); - File jobFolderInFailedJobs = new File(failedJobsDirectory); - jobFolderInFailedJobs.mkdirs(); - if(jobFolderInFailedJobs.exists()){ - return jobFolderInFailedJobs; - } - return null; - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Workspace.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Workspace.java deleted file mode 100644 index 6d3e62963..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/Workspace.java +++ /dev/null @@ -1,263 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; - -import org.apache.commons.lang.math.NumberUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.google.common.io.Files; - -/** - * All functionalities for creating the workspace, job folder, status file,
- * json input file, script file and input file for Slurm jobs. - * - * @author msff2 - * - */ -public class Workspace { - private Logger LOGGER = LogManager.getLogger(Workspace.class); - public long previousTimeStamp = System.nanoTime(); - - /** - * Receives both the agent class (e.g. DFTAgent) and folder (absolute path)
- * where the workspace for the agent will be created, and returns the name
- * of workspace. - * - * @param workspaceParentPath specific to the OS. For example,
- * - on Windows an example path will be 'C:/Users/', and
- * - on Linux the path '/home/'. - * - * @param agentClass - * @return - */ - public static File getWorkspace(String workspaceParentPath, String agentClass) { - File dir = new File(workspaceParentPath); - for (File file : dir.listFiles()) { - if (file.isDirectory()) { - if (file.getName().toLowerCase().startsWith(agentClass.toLowerCase())) { - String[] tokens = file.getName().split("_"); - if (tokens.length >= 2 && tokens[tokens.length - 1].length() > 6 - && NumberUtils.isNumber(tokens[tokens.length - 1])) { - return file; - } - } - } - } - return createWorkspaceName(workspaceParentPath, agentClass); - } - - private static File createWorkspaceName(String workspaceParentPath, String agentClass) { - String workspaceName = agentClass.concat("_").concat("" + System.nanoTime()); - File workspace = Paths.get(workspaceParentPath, workspaceName).toFile(); - if (workspace.mkdir()) { - return workspace; - } - return null; - } - - protected boolean isWorkspaceAvailable(File dir, String agentClass) { - for (File file : dir.listFiles()) { - if (file.isDirectory()) { - if (file.getName().toLowerCase().startsWith(agentClass.toLowerCase())) { - return true; - } - } - } - return false; - } - - /** - * Creates the JSON formatted input file in the current job folder. - * - * @param workspaceFolder - * @param jsonInputFilePath the path where the JSON formatted input
- * file will be stored. - * @param jsonString the string that contains the the whole content
- * of the JSON formatted input. - * @return success message if the file JSON file can be created, null
- * otherwise. - * @throws IOException - */ - public String createJSONInputFile(File workspaceFolder, String jsonInputFilePath, String jsonString) throws IOException{ - BufferedWriter jsonInputFile = Utils.openBufferedWriter(jsonInputFilePath); - if(jsonInputFile == null){ - return null; - } - jsonInputFile.write(jsonString); - jsonInputFile.close(); - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - } - - /** - * Returns the absolute path to an input file identified by its extension.
- * For example, .com, .g09, .csv, .jar, etc. - * - * @param jobFolder - * @param hpcAddress - * @param inputFileExtension - * @return - */ - public String getInputFilePath(File jobFolder, String hpcAddress, String inputFileExtension){ - return jobFolder.getAbsolutePath() - .concat(File.separator) - .concat(hpcAddress) - .concat("_").concat(getTimeStampPart(jobFolder.getName())) - .concat(inputFileExtension); - } - - public String getInputFileExtension(File input) throws SlurmJobException{ - - if(input.isFile()){ - - return input.getAbsolutePath().substring(input.getAbsolutePath().lastIndexOf(".")); - - }else{ - - LOGGER.error("SlurmJobAPI: The provided input file is not a file."); - - throw new SlurmJobException("SlurmJobAPI: The provided input file is not a file."); - } - } - - /** - * Creates the job folder on the host machine where the current agent runs. - * - * @param workspacePath - * @param hpcAddress - * @param timeStamp - * @return - */ - public File createJobFolder(String workspacePath, String hpcAddress, long timeStamp){ - String jobFolder = hpcAddress.concat("_").concat("" + timeStamp); - File workspace = new File(workspacePath.concat(File.separator).concat(jobFolder)); - if(workspace.mkdir()){ - return workspace; - } - return null; - } - - public String createInputFile(String inputFileDestinationPath, File input) throws IOException{ - copyFile(input, new File(inputFileDestinationPath)); - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - } - - public String createStatusFile(File workspaceFolder, String statusFilePath, String hpcAddress) throws IOException{ - BufferedWriter statusFile = Utils.openBufferedWriter(statusFilePath); - if(statusFile == null){ - return null; - } - statusFile.write(Status.ATTRIBUTE_JOB_STATUS.getName().concat(" ")); - statusFile.write(Status.STATUS_JOB_NOT_STARTED.getName().concat("\n")); - statusFile.write(Status.ATTRIBUTE_JOB_ID.getName().concat("\n")); - statusFile.write(Status.ATTRIBUTE_AGENT_ID.getName().concat(" ")); - statusFile.write(workspaceFolder.getName().concat("\n")); - statusFile.write(Status.ATTRIBUTE_HPC_ADDRESS.getName().concat(" ")); - statusFile.write(hpcAddress.concat("\n")); - statusFile.write(Status.ATTRIBUTE_JOB_OUTPUT.getName().concat("\n")); - statusFile.close(); - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - } - - public String copyFile(String source, String destination) throws IOException{ - try{ - copyFile(new File(source), - new File(destination)); - if(destination.endsWith(".sh")){ - Utils.translateLineEndingIntoUnix(new File(destination)); - } - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - }catch(IOException e){ - e.printStackTrace(); - return null; - } - } - - /** - * Extracts the time stamp part from the job folder. For example,
- * if the job folder is named login-skylake.hpc.cam.ac.uk_428109593378500,
- * the method will return 428109593378500. - * - * @param folder - * @return - */ - public String getTimeStampPart(String folder){ - if(folder.contains("_")){ - String[] tokens = folder.split("_"); - if(tokens.length==2 && tokens[1]!=null && StringUtils.isNumeric(tokens[1])){ - return tokens[1]; - } - } - return null; - } - - /** - * Receives the jobFolder as the input and returns the absolute path to
- * to the job status file. - * - * @param jobFolder - * @return - */ - public String getStatusFilePath(File jobFolder){ - return jobFolder.getAbsolutePath().concat(File.separator).concat(Property.STATUS_FILE_NAME.getPropertyName()); - } - - /** - * Receives the jobFolder as the input and returns the absolute path to
- * to the JSON input file. - * - * @param jobFolder - * @param jsonInputFileName - * @return - */ - public String getJSONInputFilePath(File jobFolder){ - return jobFolder.getAbsolutePath().concat(File.separator).concat(Property.JSON_INPUT_FILE_NAME.getPropertyName()); - } - - - - /** - * Copy the Slurm script to the job folder to set up the current
- * job. If the method can copy the script, it returns a job set up
- * success message. Otherwise, it returns null. - * - * - * @param source - * @param destination - * @param slurmScriptFileName - * @return - * @throws IOException - */ - public String copyScriptFile(String source, String destination, String slurmScriptFileName) throws IOException{ - try{ - copyFile(new File(source), - new File(destination.concat(File.separator) - .concat(slurmScriptFileName))); - Utils.translateLineEndingIntoUnix(new File(destination.concat(File.separator) - .concat(slurmScriptFileName))); - return Status.JOB_SETUP_SUCCESS_MSG.getName(); - }catch(IOException e){ - e.printStackTrace(); - return null; - } - } - - /** - * Copy any file from the source path to the destination path. - * - * @param from the absolute path to the file which will be copied. - * @param to the absolute path to the folder where the file will be
- * copied. If user provides absolute path including the file name,
- * probably renamed or the same as the one in the from path, the method
- * will keep the same name. - * - * @throws IOException - */ - private void copyFile(File from, File to) throws IOException{ - Files.copy(from, to); - } -} diff --git a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobProperty.java b/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobProperty.java deleted file mode 100644 index fb2cf4fdd..000000000 --- a/src/main/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobProperty.java +++ /dev/null @@ -1,181 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job.configuration; - -/** - * These properties must be set by all agents that will run Slurm jobs
- * with this API. - * - * @author Feroz Farazi (msff2@cam.ac.uk) - * - */ -public class SlurmJobProperty { - private String hpcServerLoginUserName; - private String hpcServerLoginUserPassword; - private String hpcServerPrivateKey; - private String agentClass; - private String agentWorkspacePrefix; - private String agentCompletedJobsSpacePrefix; - private String agentFailedJobsSpacePrefix; - private String hpcAddress; - private String inputFileName; - private String inputFileExtension; - private String jsonInputFileName; - private String jsonFileExtension; - private String slurmScriptFileName; - private String executableFile; - private String outputFileName; - private String outputFileExtension; - private int maxNumberOfHPCJobs; - private int agentInitialDelayToStartJobMonitoring; - private int agentPeriodicActionInterval; - - public String getHpcServerLoginUserName() { - return hpcServerLoginUserName; - } - - public void setHpcServerLoginUserName(String hpcServerLoginUserName) { - this.hpcServerLoginUserName = hpcServerLoginUserName; - } - - public String getHpcServerLoginUserPassword() { - return hpcServerLoginUserPassword; - } - - public void setHpcServerLoginUserPassword(String hpcServerLoginUserPassword) { - this.hpcServerLoginUserPassword = hpcServerLoginUserPassword; - } - - public String getHpcServerPrivateKey() { - return this.hpcServerPrivateKey; - } - public void setHpcServerPrivateKey(String hpcServerPrivateKey) { - this.hpcServerPrivateKey = hpcServerPrivateKey; - } - - public String getAgentClass() { - return agentClass; - } - - public String getAgentWorkspacePrefix() { - return agentWorkspacePrefix; - } - - public String getAgentCompletedJobsSpacePrefix() { - return agentCompletedJobsSpacePrefix; - } - - public String getHpcAddress() { - return hpcAddress; - } - - public String getInputFileName() { - return inputFileName; - } - - public String getInputFileExtension() { - return inputFileExtension; - } - - public String getJsonFileExtension() { - return jsonFileExtension; - } - - public String getJsonInputFileName() { - return jsonInputFileName; - } - - public String getSlurmScriptFileName() { - return slurmScriptFileName; - } - - public String getOutputFileName() { - return outputFileName; - } - - public String getOutputFileExtension() { - return outputFileExtension; - } - - public String getExecutableFile() { - return executableFile; - } - - public void setExecutableFile(String executableFile) { - this.executableFile = executableFile; - } - - public int getMaxNumberOfHPCJobs() { - return maxNumberOfHPCJobs; - } - - public int getAgentInitialDelayToStartJobMonitoring() { - return agentInitialDelayToStartJobMonitoring; - } - - public int getAgentPeriodicActionInterval() { - return agentPeriodicActionInterval; - } - - public void setAgentClass(String agentClass) { - this.agentClass = agentClass; - } - - public void setAgentWorkspacePrefix(String agentWorkspacePrefix) { - this.agentWorkspacePrefix = agentWorkspacePrefix; - } - - public void setAgentCompletedJobsSpacePrefix(String agentCompletedJobsSpacePrefix) { - this.agentCompletedJobsSpacePrefix = agentCompletedJobsSpacePrefix; - } - - public void setHpcAddress(String hpcAddress) { - this.hpcAddress = hpcAddress; - } - - public void setInputFileName(String inputFileName) { - this.inputFileName = inputFileName; - } - - public void setInputFileExtension(String inputFileExtension) { - this.inputFileExtension = inputFileExtension; - } - - public void setJsonFileExtension(String jsonFileExtension) { - this.jsonFileExtension = jsonFileExtension; - } - - public void setJsonInputFileName(String jsonInputFileName) { - this.jsonInputFileName = jsonInputFileName; - } - - public void setSlurmScriptFileName(String slurmScriptFileName) { - this.slurmScriptFileName = slurmScriptFileName; - } - - public void setOutputFileName(String outputFileName) { - this.outputFileName = outputFileName; - } - - public void setOutputFileExtension(String outputFileExtension) { - this.outputFileExtension = outputFileExtension; - } - - public void setMaxNumberOfHPCJobs(int maxNumberOfHPCJobs) { - this.maxNumberOfHPCJobs = maxNumberOfHPCJobs; - } - - public void setAgentInitialDelayToStartJobMonitoring(int agentInitialDelayToStartJobMonitoring) { - this.agentInitialDelayToStartJobMonitoring = agentInitialDelayToStartJobMonitoring; - } - - public void setAgentPeriodicActionInterval(int agentPeriodicActionInterval) { - this.agentPeriodicActionInterval = agentPeriodicActionInterval; - } - - public String getAgentFailedJobsSpacePrefix() { - return agentFailedJobsSpacePrefix; - } - - public void setAgentFailedJobsSpacePrefix(String agentFailedJobsSpacePrefix) { - this.agentFailedJobsSpacePrefix = agentFailedJobsSpacePrefix; - } -} diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatisticsTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatisticsTest.java deleted file mode 100644 index 8e4800ba6..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/JobStatisticsTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -class JobStatisticsTest { - @TempDir() - static Path tempFolder; - private static Path fileloc; - - // Create JobstatisticsTestInputstatus.txt. This file will be deleted at the end - // of the unit test. - @BeforeAll - static void generatefile() throws IOException { - Files.createDirectories(tempFolder); - fileloc = tempFolder.resolve("jobFolder").resolve("JobStatisticsTestInputstatus.txt"); - File tempInputStatus = fileloc.toFile(); - tempInputStatus.getParentFile().mkdirs(); - try (FileWriter writer = new FileWriter(tempInputStatus)) { - - // Fill JobStatisticsTestInputstatus.txt with text. We have one JobStatus: - // completed, two JobStatus: completing, etc. - for (int i = 0; i < 1; i++) { - writer.write("JobStatus: completed\n"); - } - - for (int i = 0; i < 2; i++) { - writer.write("JobStatus: completing\n"); - } - - for (int i = 0; i < 3; i++) { - writer.write("JobStatus: failed\n"); - } - - for (int i = 0; i < 4; i++) { - writer.write("JobStatus: pending\n"); - } - - for (int i = 0; i < 5; i++) { - writer.write("JobStatus: preempted\n"); - } - - for (int i = 0; i < 6; i++) { - writer.write("JobStatus: running\n"); - } - - for (int i = 0; i < 7; i++) { - writer.write("JobStatus: suspended\n"); - } - - for (int i = 0; i < 8; i++) { - writer.write("JobStatus: stopped\n"); - } - - for (int i = 0; i < 9; i++) { - writer.write("JobStatus: error termination\n"); - } - - for (int i = 0; i < 10; i++) { - writer.write("JobStatus: not started\n"); - } - } - } - - @Test - @DisplayName("Testing calculateStatistics") - void calculateStatistics() throws IOException, URISyntaxException { - // Call JobstatisticsTestInputstatus.txt, which was created by generatefile(). - // This file will be deleted at the end of the unit test. - File tempInputStatus = fileloc.toFile(); - File input = tempInputStatus.getParentFile(); - - // Create new object And feed JobStatisticsTestInputstatus.txt as the input. The - // file has 1 job completed, 2 jobs completing, etc, hence all of the - // assertEquals must be true for the test to be successful - JobStatistics result = new JobStatistics(input); - result.calculateStatistics(fileloc.toString()); - - assertEquals(1, result.getJobsCompleted()); - assertEquals(2, result.getJobsCompleting()); - assertEquals(3, result.getJobsFailed()); - assertEquals(4, result.getJobsPending()); - assertEquals(5, result.getJobsPreempted()); - assertEquals(6, result.getJobsRunning()); - assertEquals(7, result.getJobsSuspended()); - assertEquals(8, result.getJobsStopped()); - assertEquals(9, result.getJobsErrorTerminated()); - assertEquals(10, result.getJobsNotStarted()); - } - - @Test - @DisplayName("Testing JobStatistics") - void JobStatistics() throws IOException { - - File tempInputStatus = fileloc.toFile(); - File input = tempInputStatus.getParentFile().getParentFile(); - - JobStatistics result = new JobStatistics(input); - // There are 55 jobs total (= 1 + 2 + 3 + 4 + ... + 10). 1 completed, 2 - // completing, 3 failed, 4 pending, etc. getJobsSubmitted should therefore - // return 55. - assertEquals(55, result.getJobsSubmitted()); - } -} \ No newline at end of file diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessingTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessingTest.java deleted file mode 100644 index 6278e1680..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PostProcessingTest.java +++ /dev/null @@ -1,59 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class PostProcessingTest { - private Path tempDir; - - @BeforeEach - void createTempDir() throws IOException { - // This directory will be deleted at the end of each test. - tempDir = Files.createTempDirectory("PostProcessingTest"); - } - - @AfterEach - void deleteTempDir() throws IOException { - if (null != tempDir) { - FileUtils.forceDelete(tempDir.toFile()); - } - } - - @Test - void testupdateJobOutputStatus() throws IOException { - // Create the status.txt file. - Path tempStatusFile = tempDir.resolve("jobDir").resolve("status.txt"); - Files.createDirectories(tempStatusFile.getParent()); - Files.write(tempStatusFile, "JobOutput: processed".getBytes()); - - // Not a valid job folder as not "status.txt" file. - File falseDir = tempDir.toFile(); - assertFalse(PostProcessing.updateJobOutputStatus(falseDir)); - // "status.txt" file should be detected. - File trueDir = tempStatusFile.toFile().getParentFile(); - assertTrue(PostProcessing.updateJobOutputStatus(trueDir)); - } - - @Test - void testmodifyOutputStatus() throws IOException { - // Create the status.txt file. - Path tempStatusFile = tempDir.resolve("jobDir").resolve("status.txt"); - Files.createDirectories(tempStatusFile.getParent()); - Files.write(tempStatusFile, "JobOutput: processed".getBytes()); - - String status = "Try!!"; - PostProcessing.modifyOutputStatus(tempStatusFile.toString(), status); - Files.readAllLines(tempStatusFile).forEach(line -> assertEquals("JobOutput: " + status, line)); - } -} \ No newline at end of file diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PropertyTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PropertyTest.java deleted file mode 100644 index 442a1a232..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/PropertyTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; -import junit.framework.TestCase; - -public class PropertyTest extends TestCase{ - - // test getPropertyName - public void testgetPropertyName() { - String userhome = System.getProperty("user.home"); - - assertEquals(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), userhome); - assertEquals(Property.CHK_POINT_FILE_EXTENSION.getPropertyName(),".chk"); - assertEquals(Property.STATUS_FILE_NAME.getPropertyName(),"status.txt"); - assertEquals(Property.JSON_INPUT_FILE_NAME.getPropertyName(),"input.json"); - assertEquals(Property.SLURM_SCRIPT_FILE_NAME.getPropertyName(),"Slurm.sh"); - } - - // test getValue - public void testgetValue() { - assertEquals(Property.JOB_WORKSPACE_PARENT_DIR.getValue(), 0); - assertEquals(Property.CHK_POINT_FILE_EXTENSION.getValue(), 0); - assertEquals(Property.STATUS_FILE_NAME.getValue(),0); - assertEquals(Property.JSON_INPUT_FILE_NAME.getValue(), 0); - assertEquals(Property.SLURM_SCRIPT_FILE_NAME.getValue(), 0); - } - -} \ No newline at end of file diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobExceptionTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobExceptionTest.java deleted file mode 100644 index a87f39b54..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobExceptionTest.java +++ /dev/null @@ -1,76 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; - -import org.junit.jupiter.api.Test; - -public class SlurmJobExceptionTest { - - // Create arbitrary Throwable cause (ArithmeticException: / by zero) - public Throwable createExceptionCause(int a, int b) { - try { - int c = a / b; - } catch (ArithmeticException e) { - return e; - } - return null; - } - - // Set arbitrary inputs - String message = "test message"; - Throwable cause = createExceptionCause(1, 0); - boolean enableSuppression = false; - boolean writableStackTrace = false; - - @Test - // Test SlurmJobException() should give same results as Exception() - public void testSlurmJobException_None() { - SlurmJobException testNone = new SlurmJobException(); - Exception expectedNone = new Exception(); - assertEquals(expectedNone.getMessage(), testNone.getMessage()); - } - - @Test - // Test SlurmJobException(message) should give same results as Exception(message) - public void testSlurmJobException_Message() { - SlurmJobException testMessage = new SlurmJobException(message); - Exception expectedMessage = new Exception(message); - assertEquals(expectedMessage.getMessage(), testMessage.getMessage()); - } - - @Test - // Test SlurmJobException(message, cause) should give same results as Exception(message, cause) - public void testSlurmJobException_MessageandCause() { - SlurmJobException testMessageandCause = new SlurmJobException(message, cause); - Exception expectedMessageandCause = new Exception(message, cause); - assertEquals(expectedMessageandCause.getMessage(), testMessageandCause.getMessage()); - assertEquals(expectedMessageandCause.getCause(), testMessageandCause.getCause()); - } - - @Test - // Test SlurmJobException(cause) should give same results as Exception(cause) - public void testSlurmJobException_Cause() { - SlurmJobException testCause = new SlurmJobException(cause); - Exception expectedCause = new Exception(cause); - assertEquals(expectedCause.getCause(), testCause.getCause()); - } - - @Test - // Test SlurmJobException(message, cause, enableSuppression, writableStackTrace) - public void testAll() throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { - Constructor constrct = SlurmJobException.class.getDeclaredConstructor(String.class, Throwable.class, boolean.class, boolean.class); - constrct.setAccessible(true); - SlurmJobException testAll = constrct.newInstance(message, cause, enableSuppression, writableStackTrace); - - assertEquals("test message", testAll.getMessage()); - assertEquals(cause, testAll.getCause()); - - // enableSuppression and writableStackTrace are false, should return zero length array - assertEquals(0, testAll.getSuppressed().length); - assertEquals(0, testAll.getStackTrace().length); - } - -} \ No newline at end of file diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobTest.java deleted file mode 100644 index 29df87f8f..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/SlurmJobTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import org.junit.Assert; -import org.junit.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -public class SlurmJobTest { - - SlurmJob test = new SlurmJob(); - - //Test setJobFolderName() and getJobFolderName() - @Test - public void testSetJobFolderName() { - test.setJobFolderName("Example_One"); - Assert.assertEquals("Example_One", test.getJobFolderName()); - } - - //Test getNewJobFolderName() - @Test - public void testGetNewJobFolderName() { - String folderName = "Example_Two_1234567"; - try (MockedStatic theMock = Mockito.mockStatic(Utils.class)) { - theMock.when(Utils::getTimeStamp).thenReturn(1234567L); - Assert.assertEquals(folderName, test.getNewJobFolderName("Example_Two")); - } - //Assert.assertEquals("Example_Two_".concat("" + Utils.getTimeStamp()), test.getNewJobFolderName("Example_Two")); - } -} - diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/StatusTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/StatusTest.java deleted file mode 100644 index dc6b826ca..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/StatusTest.java +++ /dev/null @@ -1,49 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import junit.framework.TestCase; - -public class StatusTest extends TestCase{ - - - public void testStatus() { - - - assertEquals(Status.STATUS_FILE.getName(),"status.txt"); - assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName(),"JobStatus:"); - assertEquals(Status.STATUS_JOB_NOT_STARTED.getName(),"not started"); - assertEquals(Status.STATUS_JOB_COMPLETED.getName(),"completed"); - assertEquals(Status.STATUS_JOB_COMPLETING.getName(),"completing"); - assertEquals(Status.STATUS_JOB_FAILED.getName(),"failed"); - assertEquals(Status.STATUS_JOB_PENDING.getName(),"pending"); - assertEquals(Status.STATUS_JOB_PREEMPTED.getName(),"preempted"); - assertEquals(Status.STATUS_JOB_RUNNING.getName(),"running"); - assertEquals(Status.STATUS_JOB_SUSPENDED.getName(),"suspended"); - assertEquals(Status.STATUS_JOB_STOPPED.getName(),"stopped"); - assertEquals(Status.STATUS_JOB_ERROR_TERMINATED.getName(),"error termination"); - assertEquals(Status.ATTRIBUTE_JOB_ID.getName(),"JobId:"); - assertEquals(Status.ATTRIBUTE_AGENT_ID.getName(),"AgentId:"); - assertEquals(Status.ATTRIBUTE_HPC_ADDRESS.getName(),"HPCAddress:"); - assertEquals(Status.ATTRIBUTE_JOB_OUTPUT.getName(),"JobOutput:"); - assertEquals(Status.OUTPUT_PROCESSED.getName(),"processed"); - assertEquals(Status.EXTENSION_SLURM_FILE.getName(),".sh"); - assertEquals(Status.EXTENSION_LOG_FILE.getName(),".log"); - assertEquals(Status.EXTENSION_JSON_FILE.getName(),".json"); - assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(),"The job request has been processed successfully."); - assertEquals(Status.JOB_SETUP_JSON_ERROR.getName(),"The JSON request is ill formatted."); - assertEquals(Status.JOB_SETUP_SLURM_SCRIPT_ERROR.getName(),"The Slurm script could not be downloaded."); - assertEquals(Status.JOB_SETUP_INPUT_FILE_ERROR.getName(),"The input file for the job could not be created."); - assertEquals(Status.JOB_SETUP_ERROR.getName(),"The requested job could not be set up, therefore, resubmit the job."); - assertEquals(Status.JOB_LOG_MSG_ERROR_TERMINATION.getName(),"error termination"); - assertEquals(Status.JOB_LOG_MSG_NORMAL_TERMINATION.getName(),"normal termination"); - assertEquals(Status.JOB_OUTPUT_FILE_EXIST_MESSAGE.getName(),"The file exist."); - assertEquals(Status.JOB_SQUEUE_STATUS_COMPLETED.getName(),"CD"); - assertEquals(Status.JOB_SQUEUE_STATUS_COMPLETING.getName(),"CG"); - assertEquals(Status.JOB_SQUEUE_STATUS_FAILED.getName(),"F"); - assertEquals(Status.JOB_SQUEUE_STATUS_PENDING.getName(),"PD"); - assertEquals(Status.JOB_SQUEUE_STATUS_PREEMPTED.getName(),"PR"); - assertEquals(Status.JOB_SQUEUE_STATUS_RUNNING.getName(),"R"); - assertEquals(Status.JOB_SQUEUE_STATUS_SUSPENDED.getName(),"S"); - assertEquals(Status.JOB_SQUEUE_STATUS_STOPPED.getName(),"ST"); - } - -} diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/UtilsTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/UtilsTest.java deleted file mode 100644 index 91d924090..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/UtilsTest.java +++ /dev/null @@ -1,986 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.lang.reflect.Field; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Stream; - -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import uk.ac.cam.cares.jps.base.slurm.job.configuration.SlurmJobProperty; - -class UtilsTest { - - private static final String TEST_TEXT = "Lorem ipsum dolor sit amet\nconsectetur adipisci elit,\nsed eiusmod tempor incidunt\n minim veniam, quis nostrum exercitationem ullam corporis suscipit\n"; - private Path TEMP_DIR; - - @BeforeEach - void createFolder() throws IOException { - TEMP_DIR = Files.createTempDirectory("UtilsTest"); - } - - @AfterEach - void cleanUp() throws IOException { - try (Stream pathTree = Files.walk(TEMP_DIR)) { - pathTree.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); - } - } - - @Test - void getTimeStampTest() { - assertNotEquals(Utils.getTimeStamp(), Utils.getTimeStamp()); - } - - @Test - void openBufferedWriterTest() throws IOException { - - File tempfile = File.createTempFile("test_file", ".txt", TEMP_DIR.toFile()); - - try (BufferedWriter bw = Utils.openBufferedWriter(tempfile.getAbsolutePath())) { - assertNotNull(bw); - } finally { - Files.deleteIfExists(tempfile.toPath()); - } - } - - @Test - void openSourceFileTest() throws IOException { - File tempfile = File.createTempFile("test_file", ".txt", TEMP_DIR.toFile()); - - try (BufferedReader br = Utils.openSourceFile(tempfile.getAbsolutePath())) { - assertNotNull(br); - } finally { - Files.deleteIfExists(tempfile.toPath()); - } - - } - - @Test - void isJobCompletedOneTest() throws IOException { - - Path jobFolder = TEMP_DIR.resolve("login-skylake.hpc.cam.ac.uk_110761971919363"); - Files.createDirectories(jobFolder); - Path statusFile = jobFolder.resolve(Status.STATUS_FILE.getName()); - - // prepare status file for JobStatus:completed - Files.write(statusFile, - (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName()).getBytes()); - // test - assertTrue(Utils.isJobCompleted(jobFolder.toFile())); - Files.delete(statusFile); - - // prepare status file for JobStatus:error termination - Files.write(statusFile, - (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()).getBytes()); - // test - assertTrue(Utils.isJobCompleted(jobFolder.toFile())); - Files.delete(statusFile); - - // prepare status file for fail - Files.write(statusFile, " ".getBytes()); - // test - assertFalse(Utils.isJobCompleted(jobFolder.toFile())); - Files.delete(statusFile); - } - - @Test - public void isJobCompletedTwoTest() throws IOException { - - SlurmJobProperty slurmJobProperty = new SlurmJobProperty(); - slurmJobProperty.setAgentClass("UnitTestAgent"); - slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs"); - slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs"); - - File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentClass()); - Path jobFolder_path = taskspace.toPath().resolve("login-skylake.hpc.cam.ac.uk_110761971919363"); - Path status_path = jobFolder_path.resolve(Status.STATUS_FILE.getName()); - - // JobStatus:completed and JobOutput:processed - Files.createDirectories(jobFolder_path); - - Files.write(status_path, (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n" - + Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName()).getBytes()); - File jobFolder = jobFolder_path.toFile(); - String[] src1 = jobFolder.list(); - File completedJobsDir = Paths.get(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentCompletedJobsSpacePrefix() + taskspace.getName(), - jobFolder.getName()) - .toFile(); - - assertTrue(Utils.isJobCompleted(jobFolder, slurmJobProperty)); - assertTrue(completedJobsDir.isDirectory()); - List completedFiles = Arrays.asList(completedJobsDir.list()); - for (String file : src1) { - assertTrue(completedFiles.contains(file)); - } - assertFalse(jobFolder.exists()); - - // JobStatus:completed - Files.createDirectories(jobFolder_path); - Files.write(status_path, - (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n").getBytes()); - - assertTrue(Utils.isJobCompleted(jobFolder, slurmJobProperty)); - Files.delete(status_path); - - // JobStatus: error termination - Files.write(status_path, - (Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()).getBytes()); - String[] src2 = jobFolder.list(); - - File FailedJobsDir = Paths.get(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentFailedJobsSpacePrefix() + taskspace.getName(), - jobFolder.getName()) - .toFile(); - - assertTrue(Utils.isJobCompleted(jobFolder, slurmJobProperty)); - assertTrue(FailedJobsDir.isDirectory()); - List failedFiles = Arrays.asList(FailedJobsDir.list()); - for (String file : src2) { - assertTrue(failedFiles.contains(file)); - } - assertFalse(jobFolder.exists()); - - // false scenario - Files.createDirectories(jobFolder_path); - Files.write(status_path, " ".getBytes()); - assertFalse(Utils.isJobCompleted(jobFolder, slurmJobProperty)); - - FileUtils.forceDelete(taskspace); - FileUtils.forceDelete(completedJobsDir.getParentFile()); - FileUtils.forceDelete(FailedJobsDir.getParentFile()); - } - - @Test - void isJobErroneouslyCompletedOneTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for Jobstatus:error termination - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobErroneouslyCompleted(jobFolder)); - statusfile1.delete(); - - // prepare status file for fail - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(" "); - bw2.close(); - // test - assertFalse(Utils.isJobErroneouslyCompleted(jobFolder)); - - FileUtils.forceDelete(taskspace); - - } - - @Test - void isJobErroneouslyCompletedTwoTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for Jobstatus:error termination - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobErroneouslyCompleted(status_path)); - statusfile1.delete(); - - // prepare status file for fail - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(" "); - bw2.close(); - // test - assertFalse(Utils.isJobErroneouslyCompleted(status_path)); - - FileUtils.forceDelete(taskspace); - } - - @Test - void isJobRunningOneTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for JobStatus:running - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_RUNNING.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobRunning(jobFolder)); - statusfile1.delete(); - - // prepare status file for JobStatus:completing - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETING.getName()); - bw2.close(); - // test - assertTrue(Utils.isJobRunning(jobFolder)); - statusfile2.delete(); - - // prepare status file for JobStatus:pending - File statusfile3 = new File(status_path); - BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3)); - bw3.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_PENDING.getName()); - bw3.close(); - // test - assertTrue(Utils.isJobRunning(jobFolder)); - statusfile3.delete(); - - // prepare status file for fail - File statusfile4 = new File(status_path); - BufferedWriter bw4 = new BufferedWriter(new FileWriter(statusfile4)); - bw4.write(" "); - bw4.close(); - // test - assertFalse(Utils.isJobRunning(jobFolder)); - - FileUtils.forceDelete(taskspace); - - } - - @Test - void isJobRunningTwoTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for JobStatus:running - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_RUNNING.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobRunning(status_path)); - statusfile1.delete(); - - // prepare status file for JobStatus:completing - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETING.getName()); - bw2.close(); - // test - assertTrue(Utils.isJobRunning(status_path)); - statusfile2.delete(); - - // prepare status file for JobStatus:pending - File statusfile3 = new File(status_path); - BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3)); - bw3.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_PENDING.getName()); - bw3.close(); - // test - assertTrue(Utils.isJobRunning(status_path)); - statusfile3.delete(); - - // prepare status file for fail - File statusfile4 = new File(status_path); - BufferedWriter bw4 = new BufferedWriter(new FileWriter(statusfile4)); - bw4.write(" "); - bw4.close(); - // test - assertFalse(Utils.isJobRunning(status_path)); - - FileUtils.forceDelete(taskspace); - } - - @Test - void isJobNotStartedOneTest() throws IOException, NoSuchFieldException, IllegalAccessException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for JobStatus:not started - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_NOT_STARTED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobNotStarted(jobFolder)); - statusfile1.delete(); - - // False scenario - 1 - Utils utils = new Utils(); - Field isStatusFileOpen = utils.getClass().getDeclaredField("isStatusFileOpen"); - isStatusFileOpen.setAccessible(true); - isStatusFileOpen.setBoolean(isStatusFileOpen, true); - assertFalse(Utils.isJobNotStarted(jobFolder)); - isStatusFileOpen.setBoolean(isStatusFileOpen, false); - - // False scenario - 2 - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(" "); - bw2.close(); - // test - assertFalse(Utils.isJobNotStarted(jobFolder)); - - FileUtils.forceDelete(taskspace); - } - - @Test - void isJobNotStartedTwoTest() throws IOException, NoSuchFieldException, IllegalAccessException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for JobStatus:not started - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_NOT_STARTED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobNotStarted(status_path)); - statusfile1.delete(); - - // False scenario 1 - Utils utils = new Utils(); - Field isStatusFileOpen = utils.getClass().getDeclaredField("isStatusFileOpen"); - isStatusFileOpen.setAccessible(true); - isStatusFileOpen.setBoolean(isStatusFileOpen, true); - assertFalse(Utils.isJobNotStarted(status_path)); - isStatusFileOpen.setBoolean(isStatusFileOpen, false); - - // False scenario 2 - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(" "); - bw2.close(); - // test - assertFalse(Utils.isJobNotStarted(status_path)); - - FileUtils.forceDelete(taskspace); - } - - @Test - void isJobFinishedOneTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for JobStatus:completed - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobFinished(jobFolder, status_path)); - statusfile1.delete(); - - // prepare status file for JobStatus:error termination - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()); - bw2.close(); - // test - assertTrue(Utils.isJobFinished(jobFolder, status_path)); - statusfile2.delete(); - - // prepare status file for fail - File statusfile3 = new File(status_path); - BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3)); - bw3.write(" "); - bw3.close(); - // test - assertFalse(Utils.isJobFinished(jobFolder, status_path)); - FileUtils.forceDelete(taskspace); - } - - @Test - void isJobFinishedTwoTest() throws IOException { - - SlurmJobProperty slurmJobProperty = new SlurmJobProperty(); - slurmJobProperty.setAgentClass("UnitTestAgent"); - slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs"); - slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs"); - - File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentClass()); - ; - String jobFolder_path = taskspace.getAbsolutePath() + File.separator - + "login-skylake.hpc.cam.ac.uk_110761971919363"; - String status_path = jobFolder_path + File.separator + Status.STATUS_FILE.getName(); - - // JobStatus:completed and JobOutput:processed - File jobFolder1 = new File(jobFolder_path); - jobFolder1.mkdirs(); - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n" - + Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName()); - bw1.close(); - String[] src1 = jobFolder1.list(); - - File completedJobsDir_parent = new File( - Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator) - .concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(taskspace.getName())); - File completedJobsDir = new File(completedJobsDir_parent + File.separator + jobFolder1.getName()); - - assertTrue(Utils.isJobFinished(jobFolder1, status_path, slurmJobProperty)); - assertTrue(completedJobsDir.isDirectory()); - for (String file : src1) { - assertTrue(Arrays.asList(completedJobsDir.list()).contains(file)); - } - assertFalse(jobFolder1.exists()); - - // JobStatus:completed - File jobFolder2 = new File(jobFolder_path); - jobFolder2.mkdirs(); - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_COMPLETED.getName() + "\n"); - bw2.close(); - - assertTrue(Utils.isJobFinished(jobFolder2, status_path, slurmJobProperty)); - statusfile2.delete(); - - // JobStatus: error termination - File statusfile3 = new File(status_path); - BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3)); - bw3.write(Status.ATTRIBUTE_JOB_STATUS.getName() + Status.STATUS_JOB_ERROR_TERMINATED.getName()); - bw3.close(); - String[] src2 = jobFolder2.list(); - - File FailedJobsDir_parent = new File(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator) - .concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(taskspace.getName())); - File FailedJobsDir = new File(FailedJobsDir_parent + File.separator + jobFolder2.getName()); - - assertTrue(Utils.isJobFinished(jobFolder2, status_path, slurmJobProperty)); - assertTrue(FailedJobsDir.isDirectory()); - for (String file : src2) { - assertTrue(Arrays.asList(FailedJobsDir.list()).contains(file)); - } - assertFalse(jobFolder2.exists()); - - // false scenario - File jobFolder3 = new File(jobFolder_path); - jobFolder3.mkdirs(); - File statusfile4 = new File(status_path); - BufferedWriter bw4 = new BufferedWriter(new FileWriter(statusfile4)); - bw4.write(" "); - bw4.close(); - assertFalse(Utils.isJobFinished(jobFolder3, status_path, slurmJobProperty)); - - FileUtils.forceDelete(taskspace); - FileUtils.forceDelete(completedJobsDir_parent); - FileUtils.forceDelete(FailedJobsDir_parent); - - } - - @Test - void isJobPostProcessedTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for Jobstatus:error termination - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobPostProcessed(jobFolder, status_path)); - statusfile1.delete(); - - // prepare status file for fail - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(" "); - bw2.close(); - // test - assertFalse(Utils.isJobPostProcessed(jobFolder, status_path)); - - FileUtils.forceDelete(taskspace); - - } - - @Test - void isJobOutputProcessedOneTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for JobOutput:processed - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobOutputProcessed(jobFolder)); - statusfile1.delete(); - - // prepare status file for fail - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(" "); - bw2.close(); - // test - assertFalse(Utils.isJobOutputProcessed(jobFolder)); - - FileUtils.forceDelete(taskspace); - } - - @Test - void isJobOutputProcessedTwoTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - // prepare status file for JobOutput:processed - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + Status.OUTPUT_PROCESSED.getName()); - bw1.close(); - // test - assertTrue(Utils.isJobOutputProcessed(status_path)); - statusfile1.delete(); - - // prepare status file for fail - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(" "); - bw2.close(); - // test - assertFalse(Utils.isJobOutputProcessed(status_path)); - FileUtils.forceDelete(taskspace); - } - - @Test - void getJobIdTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - String tmpstr = Status.ATTRIBUTE_JOB_ID.getName() + " 28082086"; - - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(tmpstr); - bw1.close(); - - String tmpid = null; - // extract job id - String tokens[] = tmpstr.trim().toLowerCase().split(":"); - if (tokens.length >= 2 && tokens[1].trim().length() > 0) - tmpid = tokens[1].trim(); - - assertEquals(tmpid, Utils.getJobId(status_path)); - statusfile1.delete(); - - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(Status.ATTRIBUTE_JOB_ID.getName()); - bw2.close(); - - assertNull(Utils.getJobId(status_path)); - - FileUtils.forceDelete(taskspace); - } - - @Test - void addJobIdTest() throws IOException { - - Path taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609"); - File jobFolder = taskspace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363").toFile(); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - String jobid = "28082086"; - - // prepare status file for addobId method with JobStatus and JobId - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + "\n" + Status.ATTRIBUTE_JOB_ID.getName()); - bw1.close(); - - Utils.addJobId(status_path, jobid); - - // test contents of status file - BufferedReader br1 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8")); - String line; - while ((line = br1.readLine()) != null) { - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) - assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + Status.STATUS_JOB_RUNNING.getName(), line); - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName())) - assertEquals(Status.ATTRIBUTE_JOB_ID.getName() + " " + jobid, line); - - } - br1.close(); - statusfile1.delete(); - - // prepare status file for addobId method with only JobId - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(Status.ATTRIBUTE_JOB_ID.getName()); - bw2.close(); - - Utils.addJobId(status_path, jobid); - - // test contents of status file - BufferedReader br2 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8")); - while ((line = br2.readLine()) != null) { - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) - assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + Status.STATUS_JOB_RUNNING.getName(), line); - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName())) - assertEquals(Status.ATTRIBUTE_JOB_ID.getName() + " " + jobid, line); - } - br2.close(); - statusfile2.delete(); - - // prepare status file for addobId method without anything - File statusfile3 = new File(status_path); - BufferedWriter bw3 = new BufferedWriter(new FileWriter(statusfile3)); - bw3.close(); - - Utils.addJobId(status_path, jobid); - - // test contents of status file - BufferedReader br3 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8")); - while ((line = br3.readLine()) != null) { - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) - assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + Status.STATUS_JOB_RUNNING.getName(), line); - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_ID.getName())) - assertEquals(Status.ATTRIBUTE_JOB_ID.getName() + " " + jobid, line); - } - br3.close(); - } - - @Test - void modifyStatusTest() throws IOException { - - Path taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609"); - File jobFolder = taskspace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363").toFile(); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - String status1 = Status.STATUS_JOB_RUNNING.getName(); - String status2 = Status.STATUS_JOB_COMPLETED.getName(); - - // test for modifyStatus completed -> running - File statusfile1 = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile1)); - bw1.write(Status.ATTRIBUTE_JOB_STATUS.getName() + status2 + "\n"); - bw1.close(); - - Utils.modifyStatus(status_path, status1); - - BufferedReader br1 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8")); - String line; - while ((line = br1.readLine()) != null) { - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) - assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + status1, line); - - } - br1.close(); - statusfile1.delete(); - - // test for modifyStatus running -> completed - File statusfile2 = new File(status_path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(statusfile2)); - bw2.write(Status.ATTRIBUTE_JOB_STATUS.getName() + status1 + "\n"); - bw2.close(); - - Utils.modifyStatus(status_path, status2); - - BufferedReader br2 = new BufferedReader(new InputStreamReader(new FileInputStream(status_path), "UTF-8")); - while ((line = br2.readLine()) != null) { - if (line.trim().startsWith(Status.ATTRIBUTE_JOB_STATUS.getName())) - assertEquals(Status.ATTRIBUTE_JOB_STATUS.getName() + " " + status2, line); - - } - br2.close(); - } - - @Test - void getStatusFileTest() throws IOException { - - Path taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609"); - File jobFolder = taskspace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363").toFile(); - jobFolder.mkdirs(); - File jobFolder_null = new File(""); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - File statusfile = new File(status_path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(statusfile)); - bw1.close(); - assertEquals(status_path, Utils.getStatusFile(jobFolder).getAbsolutePath()); - assertNull(Utils.getStatusFile(jobFolder_null)); - } - - @Test - void getLogFilePathOnHPCTest() throws UnknownHostException { - - String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363"; - String userName = "hpcServerUserName"; - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - String hpcAddress = "login-skylake.hpc.cam.ac.uk"; - InetAddress address; - address = InetAddress.getLocalHost(); - String MachineAddress = address.toString().replace("/", "_"); - - assertEquals("/home/".concat(userName).concat("/").concat(taskspace.getName()).concat("/") - .concat(runningJob.replace(hpcAddress, MachineAddress)).concat("/") - .concat(runningJob.replace(hpcAddress, MachineAddress)).concat(Status.EXTENSION_LOG_FILE.getName()), - Utils.getLogFilePathOnHPC(runningJob, userName, taskspace, hpcAddress)); - } - - @Test - void getOutputFilePathOnHPCTest() throws UnknownHostException { - - String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363"; - String userName = "hpcServerUserName"; - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - String hpcAddress = "login-skylake.hpc.cam.ac.uk"; - String outputFileNameWithExtension = "output.log"; - InetAddress address; - address = InetAddress.getLocalHost(); - String MachineAddress = address.toString().replace("/", "_"); - - assertEquals("/home/".concat(userName).concat("/").concat(taskspace.getName()).concat("/") - .concat(runningJob.replace(hpcAddress, MachineAddress)).concat("/").concat(outputFileNameWithExtension), - Utils.getOutputFilePathOnHPC(runningJob, userName, taskspace, hpcAddress, - outputFileNameWithExtension)); - } - - @Test - void getJobFolderPathOnHPCTest() throws UnknownHostException { - String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363"; - String userName = "hpcServerUserName"; - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - String hpcAddress = "login-skylake.hpc.cam.ac.uk"; - InetAddress address; - address = InetAddress.getLocalHost(); - String MachineAddress = address.toString().replace("/", "_"); - assertEquals("/home/".concat(userName).concat("/").concat(taskspace.getName()).concat("/") - .concat(runningJob.replace(hpcAddress, MachineAddress)), - Utils.getJobFolderPathOnHPC(runningJob, userName, taskspace, hpcAddress)); - - } - - @Test - void getJobOutputFilePathOnAgentPCTest() { - - String runningJob = "login-skylake.hpc.cam.ac.uk_110761971919363"; - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - String outputFileName = "output"; - String outputFileExtension = ".log"; - assertEquals(taskspace.getAbsolutePath().concat(File.separator).concat(runningJob).concat(File.separator) - .concat(outputFileName).concat(outputFileExtension), - Utils.getJobOutputFilePathOnAgentPC(runningJob, taskspace, outputFileName, outputFileExtension)); - } - - @Test - void isErrorTerminationTest() throws IOException { - - File taskspace = TEMP_DIR.resolve("UnitTestAgent_435827288195609").toFile(); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String path = jobFolder.getAbsolutePath() + File.separator + "logFileOnAgentPC"; - - File logfile1 = new File(path); - BufferedWriter bw1 = new BufferedWriter(new FileWriter(logfile1)); - bw1.write(Status.JOB_LOG_MSG_ERROR_TERMINATION.getName().toLowerCase()); - bw1.close(); - assertTrue(Utils.isErrorTermination(path)); - logfile1.delete(); - - File logfile2 = new File(path); - BufferedWriter bw2 = new BufferedWriter(new FileWriter(logfile2)); - bw2.write(" "); - bw2.close(); - assertFalse(Utils.isErrorTermination(path)); - - FileUtils.forceDelete(taskspace); - } - - @Test - void getMachineAddressTest() throws UnknownHostException { - InetAddress address; - address = InetAddress.getLocalHost(); - String MachineAddress = address.toString().replace("/", "_"); - assertEquals(MachineAddress, Utils.getMachineAddress()); - } - - @Test - void translateLineEndingIntoUnixTest() throws IOException { - Path src = TEMP_DIR.resolve("src.txt"); - Path dest = TEMP_DIR.resolve("dest.txt"); - - Files.write(src, TEST_TEXT.replace("\n", "\r\n").getBytes()); - Files.write(dest, TEST_TEXT.getBytes()); - - Utils.translateLineEndingIntoUnix(src.toFile()); - - assertTrue(FileUtils.contentEquals(dest.toFile(), src.toFile())); - } - - @Test - void moveToCompletedJobsFolderTest() throws IOException { - - SlurmJobProperty slurmJobProperty = new SlurmJobProperty(); - slurmJobProperty.setAgentClass("UnitTestAgent"); - slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs"); - slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs"); - - File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentClass()); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - File statusfile = new File(status_path); - BufferedWriter bw = new BufferedWriter(new FileWriter(statusfile)); - bw.write("XYZ"); - bw.close(); - String[] src = jobFolder.list(); - - File completedJobsDir_parent = new File( - Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator) - .concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(taskspace.getName())); - File completedJobsDir = new File(completedJobsDir_parent + File.separator + jobFolder.getName()); - - Utils.moveToCompletedJobsFolder(jobFolder, slurmJobProperty); - assertTrue(completedJobsDir.isDirectory()); - for (String file : src) { - assertTrue(Arrays.asList(completedJobsDir.list()).contains(file)); - } - assertFalse(jobFolder.exists()); - - FileUtils.forceDelete(taskspace); - FileUtils.forceDelete(completedJobsDir_parent); - } - - @Test - void getCompletedJobsDirectoryTest() throws IOException { - - SlurmJobProperty slurmJobProperty = new SlurmJobProperty(); - slurmJobProperty.setAgentClass("UnitTestAgent"); - slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs"); - slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs"); - - File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentClass()); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - File completedJobsDir_parent = new File( - Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator) - .concat(slurmJobProperty.getAgentCompletedJobsSpacePrefix()).concat(taskspace.getName())); - File completedJobsDir = new File(completedJobsDir_parent + File.separator + jobFolder.getName()); - - assertEquals(completedJobsDir, Utils.getCompletedJobsDirectory(jobFolder, slurmJobProperty)); - FileUtils.forceDelete(taskspace); - FileUtils.forceDelete(completedJobsDir_parent); - } - - @Test - void moveToFailedJobsFolderTest() throws IOException { - - SlurmJobProperty slurmJobProperty = new SlurmJobProperty(); - slurmJobProperty.setAgentClass("UnitTestAgent"); - slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs"); - slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs"); - - File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentClass()); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - String status_path = jobFolder.getAbsolutePath() + File.separator + Status.STATUS_FILE.getName(); - - File statusfile = new File(status_path); - BufferedWriter bw = new BufferedWriter(new FileWriter(statusfile)); - bw.write("XYZ"); - bw.close(); - String[] src = jobFolder.list(); - - File FailedJobsDir_parent = new File(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator) - .concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(taskspace.getName())); - File FailedJobsDir = new File(FailedJobsDir_parent + File.separator + jobFolder.getName()); - - Utils.moveToFailedJobsFolder(jobFolder, slurmJobProperty); - assertTrue(FailedJobsDir.isDirectory()); - for (String file : src) { - assertTrue(Arrays.asList(FailedJobsDir.list()).contains(file)); - } - assertFalse(jobFolder.exists()); - - FileUtils.forceDelete(taskspace); - FileUtils.forceDelete(FailedJobsDir_parent); - - } - - @Test - void getFailedJobsDirectoryTest() throws IOException { - - SlurmJobProperty slurmJobProperty = new SlurmJobProperty(); - slurmJobProperty.setAgentClass("UnitTestAgent"); - slurmJobProperty.setAgentCompletedJobsSpacePrefix("CompletedJobs"); - slurmJobProperty.setAgentFailedJobsSpacePrefix("FailedJobs"); - - File taskspace = Workspace.getWorkspace(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName(), - slurmJobProperty.getAgentClass()); - File jobFolder = new File(taskspace + File.separator + "login-skylake.hpc.cam.ac.uk_110761971919363"); - jobFolder.mkdirs(); - File FailedJobsDir_parent = new File(Property.JOB_WORKSPACE_PARENT_DIR.getPropertyName().concat(File.separator) - .concat(slurmJobProperty.getAgentFailedJobsSpacePrefix()).concat(taskspace.getName())); - File FailedJobsDir = new File(FailedJobsDir_parent + File.separator + jobFolder.getName()); - - assertEquals(FailedJobsDir, Utils.getFailedJobsDirectory(jobFolder, slurmJobProperty)); - FileUtils.forceDelete(taskspace); - FileUtils.forceDelete(FailedJobsDir_parent); - - } -} diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/WorkspaceTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/WorkspaceTest.java deleted file mode 100644 index 299d143b7..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/WorkspaceTest.java +++ /dev/null @@ -1,248 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.stream.Stream; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class WorkspaceTest { - - private static final String TEST_TEXT = "Lorem ipsum dolor sit amet\nconsectetur adipisci elit,\nsed eiusmod tempor incidunt\n minim veniam, quis nostrum exercitationem ullam corporis suscipit\n"; - private Path TEMP_DIR; - - @BeforeEach - void createFolder() throws IOException { - TEMP_DIR = Files.createTempDirectory("WorkspaceTest"); - } - - @AfterEach - void cleanUp() throws IOException { - try (Stream pathTree = Files.walk(TEMP_DIR)) { - pathTree.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); - } - } - - @Test - void getExistingWorkspaceTest() throws IOException { - String agentClass = "UnitTestAgent"; - Path workspace = TEMP_DIR.resolve(agentClass + "_" + System.nanoTime()); - Files.createDirectories(workspace); - - assertEquals(workspace.toAbsolutePath(), - Workspace.getWorkspace(TEMP_DIR.toString(), agentClass).toPath().toAbsolutePath()); - - Files.delete(workspace); - } - - @Test - void getNewWorkspaceTest() throws IOException { - String agentClass = "UnitTestAgent"; - assertEquals(TEMP_DIR.resolve(agentClass).toString(), - Workspace.getWorkspace(TEMP_DIR.toString(), agentClass).getAbsolutePath().split("_")[0]); - } - - @Test - void isWorkspaceAvailableTestTwo() - throws IllegalAccessException, InvocationTargetException, NoSuchMethodException, IOException { - - String agentClass = "UnitTestAgent"; - Path ws = TEMP_DIR.resolve(agentClass + "_" + System.nanoTime()); - Files.createDirectories(ws); - - Workspace workspace = new Workspace(); - - assertTrue(workspace.isWorkspaceAvailable(TEMP_DIR.toFile(), agentClass)); - Files.delete(ws); - assertFalse(workspace.isWorkspaceAvailable(TEMP_DIR.toFile(), agentClass)); - } - - @Test - void createJSONInputFileTest() throws IOException { - - Path workspaceFolder = TEMP_DIR; - Path jsonInputFilePath = workspaceFolder.resolve("input.json").toAbsolutePath(); - Path jsonInputFilePath_temp = workspaceFolder.resolve("input_temp.json").toAbsolutePath(); - Workspace workspace = new Workspace(); - - Files.write(jsonInputFilePath_temp, TEST_TEXT.getBytes()); - - assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(), - workspace.createJSONInputFile(workspaceFolder.toFile(), jsonInputFilePath.toString(), TEST_TEXT)); - assertTrue(FileUtils.contentEquals(jsonInputFilePath.toFile(), jsonInputFilePath_temp.toFile())); - } - - @Test - void getInputFilePathTest() throws IOException { - - Path taskSpace = TEMP_DIR.resolve("UnitTestAgent_435827288195609"); - Path jobFolder = taskSpace.resolve("login-skylake.hpc.cam.ac.uk_110761971919363"); - Files.createDirectories(jobFolder); - String hpcAddress = "login-skylake.hpc.cam.ac.uk"; - String inputFileExtension = ".com"; - String[] tokens = jobFolder.getFileName().toString().split("_"); - String timeStampPart = null; - if (tokens.length == 2 && tokens[1] != null && StringUtils.isNumeric(tokens[1])) - timeStampPart = tokens[1]; - Workspace workspace = new Workspace(); - assertEquals( - jobFolder.resolve(hpcAddress + "_" + timeStampPart + inputFileExtension).toAbsolutePath().toString(), - workspace.getInputFilePath(jobFolder.toFile(), hpcAddress, inputFileExtension)); - } - - @Test - void getInputFileExtension() throws SlurmJobException, IOException { - - Path input = TEMP_DIR.resolve("input.csv"); - - Files.createFile(input); - - Workspace workspace = new Workspace(); - - assertEquals(input.toString().substring(input.toString().lastIndexOf('.')), - workspace.getInputFileExtension(input.toFile())); - } - - @Test - void createJobFolderTest() throws IOException { - Path workspacePath = TEMP_DIR; - String hpcAddress = "login-skylake.hpc.cam.ac.uk"; - long timeStamp = System.nanoTime(); - Workspace workspace = new Workspace(); - Path jobFolder = workspacePath.resolve(hpcAddress + "_" + timeStamp); - - assertEquals(jobFolder.toAbsolutePath(), - workspace.createJobFolder(workspacePath.toString(), hpcAddress, timeStamp).toPath().toAbsolutePath()); - assertNull(workspace.createJobFolder(workspacePath.toString(), hpcAddress, timeStamp)); - } - - @Test - void createInputFileTest() throws IOException { - - Path inputFileDestinationPath = TEMP_DIR.resolve("input_dest"); - Path input = TEMP_DIR.resolve("input_src"); - Workspace workspace = new Workspace(); - - Files.write(input, TEST_TEXT.getBytes()); - - assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(), - workspace.createInputFile(inputFileDestinationPath.toString(), input.toFile())); - assertTrue(FileUtils.contentEquals(input.toFile(), inputFileDestinationPath.toFile())); - } - - @Test - void createStatusFileTest() throws IOException { - - Path workspaceFolder = TEMP_DIR; - Path statusFilePath = workspaceFolder.resolve(Status.STATUS_FILE.getName()); - String hpcAddress = "login-skylake.hpc.cam.ac.uk"; - Path tempFile = workspaceFolder.resolve("temp.txt"); - Workspace workspace = new Workspace(); - - try (BufferedWriter bw = new BufferedWriter(new FileWriter(tempFile.toFile()))) { - bw.write(Status.ATTRIBUTE_JOB_STATUS.getName() + " "); - bw.write(Status.STATUS_JOB_NOT_STARTED.getName() + "\n"); - bw.write(Status.ATTRIBUTE_JOB_ID.getName() + "\n"); - bw.write(Status.ATTRIBUTE_AGENT_ID.getName() + " "); - bw.write(workspaceFolder.getFileName() + "\n"); - bw.write(Status.ATTRIBUTE_HPC_ADDRESS.getName() + " "); - bw.write(hpcAddress + "\n"); - bw.write(Status.ATTRIBUTE_JOB_OUTPUT.getName() + "\n"); - } - - assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(), - workspace.createStatusFile(workspaceFolder.toFile(), statusFilePath.toString(), hpcAddress)); - assertTrue(FileUtils.contentEquals(statusFilePath.toFile(), tempFile.toFile())); - } - - @Test - void copyFileTestOne() throws IOException { - - Path destination = TEMP_DIR.resolve("dest"); - Path source = TEMP_DIR.resolve("src"); - Path temp = TEMP_DIR.resolve("temp"); - Workspace workspace = new Workspace(); - - Files.write(source, TEST_TEXT.getBytes()); - - assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(), - workspace.copyFile(source.toString(), destination.toString())); - assertTrue(FileUtils.contentEquals(source.toFile(), destination.toFile())); - - Files.delete(destination); - - Path destination_sh = TEMP_DIR.resolve("dest.sh"); - Files.write(temp, TEST_TEXT.getBytes()); - - assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(), - workspace.copyFile(source.toString(), destination_sh.toString())); - assertTrue(FileUtils.contentEquals(temp.toFile(), destination_sh.toFile())); - } - - @Test - void getTimeStampPartTest() { - - String folder = "login-skylake.hpc.cam.ac.uk_428109593378500"; - String folder_null = ""; - Workspace workspace = new Workspace(); - String[] tokens = folder.split("_"); - assertEquals(tokens[1], workspace.getTimeStampPart(folder)); - assertNull(workspace.getTimeStampPart(folder_null)); - } - - @Test - void getStatusFilePathTest() { - - Path jobFolder = TEMP_DIR; - Workspace workspace = new Workspace(); - String expected = jobFolder.resolve(Property.STATUS_FILE_NAME.getPropertyName()).toAbsolutePath().toString(); - assertEquals(expected, workspace.getStatusFilePath(jobFolder.toFile())); - } - - @Test - void getJSONInputFilePathTest() { - - Path jobFolder = TEMP_DIR; - Workspace workspace = new Workspace(); - String expected = jobFolder.resolve(Property.JSON_INPUT_FILE_NAME.getPropertyName()).toAbsolutePath() - .toString(); - assertEquals(expected, workspace.getJSONInputFilePath(jobFolder.toFile())); - } - - @Test - void copyScriptFileTest() throws IOException { - - Path source = TEMP_DIR.resolve("source"); - Path dest = TEMP_DIR.resolve("dest"); - Files.createDirectories(dest); - - String slurmScriptFileName = "slurmscript.sh"; - Path temp = TEMP_DIR.resolve("temp"); - Workspace workspace = new Workspace(); - - Files.write(source, TEST_TEXT.getBytes()); - Files.write(temp, TEST_TEXT.getBytes()); - - assertEquals(Status.JOB_SETUP_SUCCESS_MSG.getName(), - workspace.copyScriptFile(source.toString(), dest.toString(), slurmScriptFileName)); - Path slurmScriptPath = dest.resolve(slurmScriptFileName); - assertTrue(FileUtils.contentEquals(temp.toFile(), slurmScriptPath.toFile())); - } -} diff --git a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobPropertyTest.java b/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobPropertyTest.java deleted file mode 100644 index 892a813a0..000000000 --- a/src/test/java/uk/ac/cam/cares/jps/base/slurm/job/configuration/SlurmJobPropertyTest.java +++ /dev/null @@ -1,142 +0,0 @@ -package uk.ac.cam.cares.jps.base.slurm.job.configuration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -import org.junit.Test; - -public class SlurmJobPropertyTest { - - SlurmJobProperty test = new SlurmJobProperty(); - - @Test - public void getHpcServerLoginUserNameTest() { - // Default value - assertNull(test.getHpcServerLoginUserName()); - // Set specific value and check whether it is set correctly - test.setHpcServerLoginUserName("user"); - assertEquals("user", test.getHpcServerLoginUserName()); - } - - @Test - public void getHpcServerLoginUserPasswordTest() { - assertNull(test.getHpcServerLoginUserPassword()); - test.setHpcServerLoginUserPassword("user"); - assertEquals("user", test.getHpcServerLoginUserPassword()); - } - - @Test - public void getAgentClassTest() { - assertNull(test.getAgentClass()); - test.setAgentClass("user"); - assertEquals("user", test.getAgentClass()); - } - - @Test - public void getAgentWorkspacePrefixTest() { - assertNull(test.getAgentWorkspacePrefix()); - test.setAgentWorkspacePrefix("user"); - assertEquals("user", test.getAgentWorkspacePrefix()); - } - - @Test - public void getAgentCompletedJobsSpacePrefixTest() { - assertNull(test.getAgentCompletedJobsSpacePrefix()); - test.setAgentCompletedJobsSpacePrefix("user"); - assertEquals("user", test.getAgentCompletedJobsSpacePrefix()); - } - - @Test - public void getHpcAddressTest() { - assertNull(test.getHpcAddress()); - test.setHpcAddress("user"); - assertEquals("user", test.getHpcAddress()); - } - - @Test - public void getInputFileNameTest() { - assertNull(test.getInputFileName()); - test.setInputFileName("user"); - assertEquals("user", test.getInputFileName()); - } - - @Test - public void getInputFileExtensionTest() { - assertNull(test.getInputFileExtension()); - test.setInputFileName("user"); - assertEquals("user", test.getInputFileName()); - } - - @Test - public void getJsonInputFileNameTest() { - assertNull(test.getJsonInputFileName()); - test.setJsonInputFileName("user"); - assertEquals("user", test.getJsonInputFileName()); - } - - @Test - public void getJsonFileExtensionTest() { - assertNull(test.getJsonFileExtension()); - test.setJsonFileExtension("user"); - assertEquals("user", test.getJsonFileExtension()); - } - - @Test - public void getSlurmScriptFileNameTest() { - assertNull(test.getSlurmScriptFileName()); - test.setSlurmScriptFileName("user"); - assertEquals("user", test.getSlurmScriptFileName()); - } - - @Test - public void getOutputFileNameTest() { - assertNull(test.getOutputFileName()); - test.setOutputFileName("user"); - assertEquals("user", test.getOutputFileName()); - } - - @Test - public void getOutputFileExtensionTest() { - assertNull(test.getOutputFileExtension()); - test.setOutputFileExtension("user"); - assertEquals("user", test.getOutputFileExtension()); - } - - @Test - public void getExecutableFileTest() { - assertNull(test.getExecutableFile()); - test.setExecutableFile("user"); - assertEquals("user", test.getExecutableFile()); - } - - @Test - public void getAgentFailedJobsSpacePrefixTest() { - assertNull(test.getAgentFailedJobsSpacePrefix()); - test.setAgentFailedJobsSpacePrefix("user"); - assertEquals("user", test.getAgentFailedJobsSpacePrefix()); - } - - - @Test - public void getMaxNumberOfHPCJobsTest() { - assertEquals(test.getMaxNumberOfHPCJobs(), 0); - test.setMaxNumberOfHPCJobs(123); - assertEquals(123, test.getMaxNumberOfHPCJobs()); - } - - @Test - public void getAgentInitialDelayToStartJobMonitoringTest() { - assertEquals(test.getAgentInitialDelayToStartJobMonitoring(), 0); - test.setAgentInitialDelayToStartJobMonitoring(123); - assertEquals(123, test.getAgentInitialDelayToStartJobMonitoring()); - } - - @Test - public void getAgentPeriodicActionIntervalTest() { - assertEquals(test.getAgentPeriodicActionInterval(), 0); - test.setAgentPeriodicActionInterval(123); - assertEquals(123, test.getAgentPeriodicActionInterval()); - - } - -} \ No newline at end of file