%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% On a cluster such as SCC, jobs that take hours or days to run to completion    %
% on occasion could suffer from system abort in mid-stream due to a variety of   %
% reasons: power failure, walltime limit, scheduled shutdown, to name a few.     %
% Rerunning jobs oftens means inevitable delay and waste of system resources.    %
% Checkpoint Restarting has long been a common technique with which researchers  %
% turn to tackle this issue. In brief, Checkpoint Restarting essentially means   %
% saving data to disk periodically so that if need be, you can restart the job   %
% where data were last saved. Checkpoint Restarting can either be dealt with     %
% through a batch scheduler (if supported) or do it the old-fashioned way by     %
% writing to disk manually yourself. On the SCC, the OGE batch scheduler         %
% checkpointing feature is not supported. A rather simple MATLAB checkpointing   %
% tool has been developed by SCV to ease the pain of manual checkpointing.       %
% This MATLAB example code demonstrates the usage of this checkpointing tool to  %
% facilitate rerunning of your job should a system abort occurs.                 %
%                                                                                %
% December 8, 2013                                                               %
% Kadin Tseng, SCV, Boston University  (kadin@bu.edu)                            %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
a = 123;          % a constant needed for restarting
n = 10;           % another such constant
Niter = 500;      % total number of iterations
frequency = 100;  % how often you want to perform checkpointing (intermediate save)

%=======================================================================================
% Before the start of the iteration loop, "check-in" each and every variable
% that should be checkpointed in the event of restarting the job

matfile = 'chkpt.mat';      % mandatory; name of checkpoint mat-file; your name choice
s = struct();               % mandatory; create struct for checkpointing variables
s = chkin(s,{'iter'});      % mandatory; iter is the name of the iteration loop index
s = chkin(s,{'frequency'}); % mandatory; frequency is the period of checkpointing, i.e.,
                            % how often you want to perform a save. A finer grain saving
                            % prevents less lost iteration but costs more because of it
s = chkin(s,{'Niter'});     % mandatory; total number of iterations
s = chkin(s,{'a' 'n' 'A'}); % a, n, A are names of variables needed saving

% continue until all variables are checked in. Note that you are only
% checking in the variables, they don't need to have been already defined

chkNames = fieldnames(s);    % the full list of variables to checkpoint
nNames = length(chkNames);   % number of variables in list
%=======================================================================================

for iter=1:Niter

   A = rand(n);    % performs computations

%=======================================================================================
% Before the end of the iteration loop, checkpoints every **frequency** steps
if mod(iter, frequency) == 0
   iter    % prints iteration number
   chkpt   % performs checkpointing (save) every **frequency** iterations
end
%=======================================================================================

end   % ending iteration loop
