Merge pull request #1 from mathworks/merge-submission-modes

Merge submission modes

Author: jmcave

README.md

@@ -13,10 +13,7 @@
     error('parallelexamples:GenericGridEngine:SubmitFcnError', ...
         'The function %s is for use with clusters created using the parcluster command.', currFilename)
 end
-if ~cluster.HasSharedFilesystem
-    error('parallelexamples:GenericGridEngine:NotSharedFileSystem', ...
-        'The function %s is for use with shared filesystems.', currFilename)
-end
+
 % Get the information about the actual cluster used
 data = cluster.getJobClusterData(job);
 if isempty(data)
@@ -36,8 +33,7 @@
     commandToRun = sprintf('qdel "%s"', schedulerID);
     dctSchedulerMessage(4, '%s: Canceling job on cluster using command:\n\t%s.', currFilename, commandToRun);
     try
-        % Make the shelled out call to run the command.
-        [cmdFailed, cmdOut] = runSchedulerCommand(commandToRun);
+        [cmdFailed, cmdOut] = runSchedulerCommand(cluster, commandToRun);
     catch err
         cmdFailed = true;
         cmdOut = err.message;
@@ -55,6 +51,21 @@
     end
 end
 
+if ~cluster.HasSharedFilesystem
+    % Only stop mirroring if we are actually mirroring
+    remoteConnection = getRemoteConnection(cluster);
+    if remoteConnection.isJobUsingConnection(job.ID)
+        dctSchedulerMessage(5, '%s: Stopping the mirror for job %d.', currFilename, job.ID);
+        try
+            remoteConnection.stopMirrorForJob(job);
+        catch err
+            warning('parallelexamples:GenericGridEngine:FailedToStopMirrorForJob', ...
+                'Failed to stop the file mirroring for job %d.\nReason: %s', ...
+                job.ID, err.getReport);
+        end
+    end
+end
+
 % Now warn about those jobs that we failed to cancel.
 erroredJobAndCauseStrings = erroredJobAndCauseStrings(~cellfun(@isempty, erroredJobAndCauseStrings));
 if ~isempty(erroredJobAndCauseStrings)

shared/cancelJobFcn.m renamed to cancelJobFcn.m

@@ -13,10 +13,7 @@
     error('parallelexamples:GenericGridEngine:SubmitFcnError', ...
         'The function %s is for use with clusters created using the parcluster command.', currFilename)
 end
-if ~cluster.HasSharedFilesystem
-    error('parallelexamples:GenericGridEngine:NotSharedFileSystem', ...
-        'The function %s is for use with shared filesystems.', currFilename)
-end
+
 % Get the information about the actual cluster used
 data = cluster.getJobClusterData(task.Parent);
 if isempty(data)
@@ -45,8 +42,7 @@
 commandToRun = sprintf('qdel "%s"', schedulerID);
 dctSchedulerMessage(4, '%s: Canceling task on cluster using command:\n\t%s.', currFilename, commandToRun);
 try
-    % Make the shelled out call to run the command.
-    [cmdFailed, cmdOut] = runSchedulerCommand(commandToRun);
+    [cmdFailed, cmdOut] = runSchedulerCommand(cluster, commandToRun);
 catch err
     cmdFailed = true;
     cmdOut = err.message;

shared/cancelTaskFcn.m renamed to cancelTaskFcn.m

@@ -17,17 +17,14 @@ function communicatingSubmitFcn(cluster, job, environmentProperties)
 
 decodeFunction = 'parallel.cluster.generic.communicatingDecodeFcn';
 
-if cluster.HasSharedFilesystem
-    error('parallelexamples:GenericGridEngine:NotNonSharedFileSystem', ...
-        'The function %s is for use with nonshared filesystems.', currFilename)
-end
-
 if ~strcmpi(cluster.OperatingSystem, 'unix')
     error('parallelexamples:GenericGridEngine:UnsupportedOS', ...
         'The function %s only supports clusters with unix OS.', currFilename)
 end
 
-remoteConnection = getRemoteConnection(cluster);
+if isprop(cluster.AdditionalProperties, 'ClusterHost')
+    remoteConnection = getRemoteConnection(cluster);
+end
 
 % Determine the debug setting. Setting to true makes the MATLAB workers
 % output additional logging. If EnableDebug is set in the cluster object's
@@ -65,11 +62,16 @@ function communicatingSubmitFcn(cluster, job, environmentProperties)
 % The job specific environment variables
 % Remove leading and trailing whitespace from the MATLAB arguments
 matlabArguments = strtrim(environmentProperties.MatlabArguments);
-% Where on the remote filesystem to store job output
-storageLocation = remoteConnection.JobStorageLocation;
-% If the RemoteJobStorageLocation ends with a space, add a slash to ensure it is respected
-if endsWith(storageLocation, ' ')
-    storageLocation = [storageLocation, fileSeparator];
+
+% Where the workers store job output
+if cluster.HasSharedFilesystem
+    storageLocation = environmentProperties.StorageLocation;
+else
+    storageLocation = remoteConnection.JobStorageLocation;
+    % If the RemoteJobStorageLocation ends with a space, add a slash to ensure it is respected
+    if endsWith(storageLocation, ' ')
+        storageLocation = [storageLocation, fileSeparator];
+    end
 end
 variables = {'PARALLEL_SERVER_DECODE_FUNCTION', decodeFunction; ...
     'PARALLEL_SERVER_STORAGE_CONSTRUCTOR', environmentProperties.StorageConstructor; ...
@@ -93,33 +95,37 @@ function communicatingSubmitFcn(cluster, job, environmentProperties)
 nonEmptyValues = cellfun(@(x) ~isempty(strtrim(x)), variables(:,2));
 variables = variables(nonEmptyValues, :);
 
-% The local job directory
+% The job directory as accessed by this machine
 localJobDirectory = cluster.getJobFolder(job);
-% How we refer to the job directory on the cluster
-remoteJobDirectory = remoteConnection.getRemoteJobLocation(job.ID, cluster.OperatingSystem);
+
+% The job directory as accessed by workers on the cluster
+if cluster.HasSharedFilesystem
+    jobDirectoryOnCluster = cluster.getJobFolderOnCluster(job);
+else
+    jobDirectoryOnCluster = remoteConnection.getRemoteJobLocation(job.ID, cluster.OperatingSystem);
+end
 % Specify the job wrapper script to use.
 % Prior to R2019a, only the SMPD process manager is supported.
 if verLessThan('matlab', '9.6') || ...
         isprop(cluster.AdditionalProperties, 'UseSmpd') && cluster.AdditionalProperties.UseSmpd
-    scriptName = 'communicatingJobWrapperSmpd.sh';
+    jobWrapperName = 'communicatingJobWrapperSmpd.sh';
     parallelEnvironment = 'matlabSmpd';
 else
-    scriptName = 'communicatingJobWrapper.sh';
+    jobWrapperName = 'communicatingJobWrapper.sh';
     parallelEnvironment = 'matlab';
 end
 % The wrapper script is in the same directory as this file
 dirpart = fileparts(mfilename('fullpath'));
-localScript = fullfile(dirpart, scriptName);
+localScript = fullfile(dirpart, jobWrapperName);
 % Copy the local wrapper script to the job directory
 copyfile(localScript, localJobDirectory);
 
-% The command that will be executed on the remote host to run the job.
-remoteScriptName = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, scriptName);
-quotedScriptName = sprintf('%s%s%s', quote, remoteScriptName, quote);
+% The script to execute on the cluster to run the job
+wrapperPath = sprintf('%s%s%s', jobDirectoryOnCluster, fileSeparator, jobWrapperName);
+quotedWrapperPath = sprintf('%s%s%s', quote, wrapperPath, quote);
 
-% Choose a file for the output. Please note that currently, JobStorageLocation refers
-% to a directory on disk, but this may change in the future.
-logFile = sprintf('%s%s%s', remoteJobDirectory, fileSeparator, sprintf('Job%d.log', job.ID));
+% Choose a file for the output
+logFile = sprintf('%s%s%s', jobDirectoryOnCluster, fileSeparator, sprintf('Job%d.log', job.ID));
 quotedLogFile = sprintf('%s%s%s', quote, logFile, quote);
 dctSchedulerMessage(5, '%s: Using %s as log file', currFilename, quotedLogFile);
 
@@ -131,43 +137,64 @@ function communicatingSubmitFcn(cluster, job, environmentProperties)
 numSlots = environmentProperties.NumberOfTasks;
 additionalSubmitArgs = sprintf('-pe %s %d', parallelEnvironment, numSlots);
 dctSchedulerMessage(4, '%s: Requesting %d slots', currFilename, numSlots);
+if cluster.NumThreads > 1
+    additionalSubmitArgs = sprintf('-binding pe linear:%d %s', cluster.NumThreads, additionalSubmitArgs);
+end
 commonSubmitArgs = getCommonSubmitArgs(cluster);
 additionalSubmitArgs = strtrim(sprintf('%s %s', additionalSubmitArgs, commonSubmitArgs));
 
 % Create a script to submit a Grid Engine job - this will be created in the job directory
 dctSchedulerMessage(5, '%s: Generating script for job.', currFilename);
-localScriptName = tempname(localJobDirectory);
-[~, scriptName] = fileparts(localScriptName);
-remoteScriptLocation = sprintf('%s%s%s%s%s', quote, remoteJobDirectory, fileSeparator, scriptName, quote);
-createSubmitScript(localScriptName, jobName, quotedLogFile, quotedScriptName, ...
+localSubmitScriptPath = tempname(localJobDirectory);
+createSubmitScript(localSubmitScriptPath, jobName, quotedLogFile, quotedWrapperPath, ...
     variables, additionalSubmitArgs);
-% Create the command to run on the remote host.
-commandToRun = sprintf('sh %s', remoteScriptLocation);
 
-% Start the mirror to copy all the job files over to the cluster
-dctSchedulerMessage(4, '%s: Starting mirror for job %d.', currFilename, job.ID);
-remoteConnection.startMirrorForJob(job);
+% Path to the submit script as seen by the cluster
+[~, submitScriptName] = fileparts(localSubmitScriptPath);
+submitScriptPathOnCluster = sprintf('%s%s%s', jobDirectoryOnCluster, fileSeparator, submitScriptName);
+quotedSubmitScriptPathOnCluster = sprintf('%s%s%s', quote, submitScriptPathOnCluster, quote);
+
+% Create the command to run on the cluster
+commandToRun = sprintf('sh %s', quotedSubmitScriptPathOnCluster);
+
+if ~cluster.HasSharedFilesystem
+    % Start the mirror to copy all the job files over to the cluster
+    dctSchedulerMessage(4, '%s: Starting mirror for job %d.', currFilename, job.ID);
+    remoteConnection.startMirrorForJob(job);
+end
 
-% Add execute permissions to shell scripts
-remoteConnection.runCommand(sprintf( ...
-    'chmod u+x %s%s*.sh', remoteJobDirectory, fileSeparator));
+if isprop(cluster.AdditionalProperties, 'ClusterHost')
+    % Add execute permissions to shell scripts
+    runSchedulerCommand(cluster, sprintf( ...
+        'chmod u+x %s%s*.sh', jobDirectoryOnCluster, fileSeparator));
+    % Convert line endings to Unix
+    runSchedulerCommand(cluster, sprintf( ...
+        'dos2unix %s%s*.sh', jobDirectoryOnCluster, fileSeparator));
+end
 
 % Now ask the cluster to run the submission command
 dctSchedulerMessage(4, '%s: Submitting job using command:\n\t%s', currFilename, commandToRun);
-% Execute the command on the remote host.
-[cmdFailed, cmdOut] = remoteConnection.runCommand(commandToRun);
+try
+    [cmdFailed, cmdOut] = runSchedulerCommand(cluster, commandToRun);
+catch err
+    cmdFailed = true;
+    cmdOut = err.message;
+end
 if cmdFailed
-    % Stop the mirroring if we failed to submit the job - this will also
-    % remove the job files from the remote location
-    % Only stop mirroring if we are actually mirroring
-    if remoteConnection.isJobUsingConnection(job.ID)
-        dctSchedulerMessage(5, '%s: Stopping the mirror for job %d.', currFilename, job.ID);
-        try
-            remoteConnection.stopMirrorForJob(job);
-        catch err
-            warning('parallelexamples:GenericGridEngine:FailedToStopMirrorForJob', ...
-                'Failed to stop the file mirroring for job %d.\nReason: %s', ...
-                job.ID, err.getReport);
+    if ~cluster.HasSharedFilesystem
+        % Stop the mirroring if we failed to submit the job - this will also
+        % remove the job files from the remote location
+        remoteConnection = getRemoteConnection(cluster);
+        % Only stop mirroring if we are actually mirroring
+        if remoteConnection.isJobUsingConnection(job.ID)
+            dctSchedulerMessage(5, '%s: Stopping the mirror for job %d.', currFilename, job.ID);
+            try
+                remoteConnection.stopMirrorForJob(job);
+            catch err
+                warning('parallelexamples:GenericGridEngine:FailedToStopMirrorForJob', ...
+                    'Failed to stop the file mirroring for job %d.\nReason: %s', ...
+                    job.ID, err.getReport);
+            end
         end
     end
     error('parallelexamples:GenericGridEngine:FailedToSubmitJob', ...
@@ -188,11 +215,16 @@ function communicatingSubmitFcn(cluster, job, environmentProperties)
 end
 
 % Store the scheduler ID for each task and the job cluster data
-% Set the cluster host and remote job storage location on the job cluster data
-jobData = struct('type', 'generic', ...
-    'RemoteHost', remoteConnection.Hostname, ...
-    'RemoteJobStorageLocation', remoteConnection.JobStorageLocation, ...
-    'HasDoneLastMirror', false);
+jobData = struct('type', 'generic');
+if isprop(cluster.AdditionalProperties, 'ClusterHost')
+    % Store the cluster host
+    jobData.RemoteHost = remoteConnection.Hostname;
+end
+if ~cluster.HasSharedFilesystem
+    % Store the remote job storage location
+    jobData.RemoteJobStorageLocation = remoteConnection.JobStorageLocation;
+    jobData.HasDoneLastMirror = false;
+end
 if verLessThan('matlab', '9.7') % schedulerID stored in job data
     jobData.ClusterJobIDs = jobIDs;
 else % schedulerID on task since 19b

nonshared/communicatingJobWrapper.sh renamed to communicatingJobWrapper.sh

@@ -1,4 +1,4 @@
-function createSubmitScript(outputFilename, jobName, quotedLogFile, quotedScriptName, ...
+function createSubmitScript(outputFilename, jobName, quotedLogFile, quotedWrapperPath, ...
     environmentVariables, additionalSubmitArgs, jobArrayString)
 % Create a script that sets the correct environment variables and then
 % executes the Grid Engine qsub command.
@@ -31,7 +31,7 @@ function createSubmitScript(outputFilename, jobName, quotedLogFile, quotedScript
 % We will forward all environment variables with this job in the call
 % to qsub
 variablesToForward = environmentVariables(:,1);
-commandToRun = getSubmitString(jobName, quotedLogFile, quotedScriptName, ...
+commandToRun = getSubmitString(jobName, quotedLogFile, quotedWrapperPath, ...
     variablesToForward, additionalSubmitArgs, jobArrayString);
 fprintf(fid, '%s\n', commandToRun);
 

nonshared/communicatingJobWrapperSmpd.sh renamed to communicatingJobWrapperSmpd.sh

@@ -0,0 +1,54 @@
+function deleteJobFcn(cluster, job)
+%DELETEJOBFCN Deletes a job on Grid Engine
+%
+% Set your cluster's PluginScriptsLocation to the parent folder of this
+% function to run it when you delete a job.
+
+% Copyright 2017-2022 The MathWorks, Inc.
+
+cancelJobFcn(cluster, job);
+
+if cluster.HasSharedFilesystem
+    % If we delete the job files before Grid Engine has actually finished
+    % processing the job, the job may be left in an 'Eqw' state and appear in
+    % the output of qstat until it is deleted with qdel. To try to prevent this
+    % from happening, wait a maximum of 20 seconds for the job to not exist
+    % according to qstat before returning.
+    % CONFIGURATION OF THIS TIME MAY BE REQUIRED.
+    maxTimeToWait = seconds(20);
+    jobCancelTime = datetime('now', 'TimeZone', 'local');
+    
+    % Store the current filename for the errors, warnings and dctSchedulerMessages
+    currFilename = mfilename;
+    
+    % Get the information about the actual cluster used
+    data = cluster.getJobClusterData(job);
+    if isempty(data)
+        % This indicates that the job has not been submitted, so just return
+        dctSchedulerMessage(1, '%s: Job cluster data was empty for job with ID %d.', currFilename, job.ID);
+        return
+    end
+    schedulerIDs = getSimplifiedSchedulerIDsForJob(job);
+    schedulerIDsString = strjoin(schedulerIDs, ',');
+    commandToRun = sprintf('qstat -j %s', schedulerIDsString);
+    
+    while (datetime('now', 'TimeZone', 'local') - jobCancelTime) < maxTimeToWait
+        dctSchedulerMessage(4, '%s: Checking job does not exist on scheduler using command:\n\t%s.', currFilename, commandToRun);
+        try
+            [cmdFailed, ~] = runSchedulerCommand(cluster, commandToRun);
+        catch err %#ok<NASGU>
+            cmdFailed = true;
+        end
+        
+        % qstat returns a non-zero error code if the jobs do not exist. This is
+        % desired behaviour.
+        if cmdFailed
+            dctSchedulerMessage(4, '%s: qstat failed. Assuming job does not exist on scheduler.', currFilename);
+            return
+        end
+        pause(1);
+    end
+    dctSchedulerMessage(4, '%s: Job still exists on scheduler despite waiting %s.', currFilename, char(maxTimeToWait));
+end
+
+end