function [step_failure, f, df_proj] = handle_step_failure(obj, f, df_proj, indx)
% Check if an update step failed and hence update current position;
    step_failure = false; % check to see whether the step should be a failure;
    if ~isfinite(f) || sum(~isfinite(df_proj))>0
        step_failure = true; % if function or its gradient is not-finite then it's failure;
    elseif obj.eval_count(indx) == 1 % new subfunction with only one evaluation
        if max(obj.eval_count) > 1 
            if f > mean(obj.hist_f(obj.eval_count>1,1)) + 3*std(obj.hist_f(obj.eval_count>1,1))
                step_failure = true; % much larger than expected then it's failure;
            end
        end
    elseif f > obj.hist_f(indx,1) % subfunction has increased in value compared to its history
        % (in this case we check if it's larger than its predicted value by enough to trigger a failure);
        f_pred = obj.get_predicted_subf(indx, obj.theta_proj); % calculate the predicted value of this subfunction;
        % subfunction exceeds its predicted value by more than average gain in other subfunctions ?
        predicted_improvement_others = obj.f_predicted_total_improvement - (obj.hist_f(indx,1) - f_pred);
        if f - f_pred > predicted_improvement_others
            step_failure = true; % then the step is failure 
        end
    end
    % Step length in both cases (Equation (16))
    if ~step_failure
        obj.step_scale = 1./obj.N + obj.step_scale .* (1. - 1./obj.N); % decay the step_scale back towards 1;
    else
        obj.step_scale = obj.step_scale / 2.; % shorten the step length;
        % subspace may be updated during the function calls so store this in the full space;
        df = (obj.P) * (df_proj);
        [f_lastpos, df_lastpos_proj] = obj.f_df_wrapper(obj.theta_prior_step, indx);
        df_lastpos = (obj.P) * (df_lastpos_proj);

        % if the function value exploded, then back it off until it's a reasonable order of magnitude before adding anything to the history;
        f_pred = obj.get_predicted_subf(indx, obj.theta_proj);
        if isfinite(obj.hist_f(indx,1))
            predicted_f_diff = abs(f_pred - obj.hist_f(indx,1));
        else
            predicted_f_diff = abs(f - f_lastpos);
        end
        if ~isfinite(predicted_f_diff) || predicted_f_diff < obj.eps
            predicted_f_diff = obj.eps;
        end

        for i_ls = 1:10
            if f - f_lastpos < 10*predicted_f_diff % failed update within an order of magnitude of the target update value
                break; % no backoff required;
            end
            % make the step length a factor of 100 shorter;
            obj.theta = 0.99.*obj.theta_prior_step + 0.01.*obj.theta;
            obj.theta_proj = (obj.P.') * (obj.theta);
            % recompute f & df at this new location;
            [f, df_proj] = obj.f_df_wrapper(obj.theta, indx);
            df = (obj.P) * (df_proj);
        end
        % move into the subspace.;
        df_proj = (obj.P.') * (df);
        df_lastpos_proj = (obj.P.') * (df_lastpos);

        if f < f_lastpos % step failed, but last position was even worse (original objective is better);
            theta_lastpos_proj = (obj.P.') * (obj.theta_prior_step);
            obj.update_history(indx, theta_lastpos_proj, f_lastpos, df_lastpos_proj); % add the newly evaluated point to the history;
        else % step failed but proposed f
            % add the change in theta & the change in gradient to the history for this subfunction before failing over to the last position;
            if isfinite(f) && sum(~isfinite(df_proj))==0
                obj.update_history(indx, obj.theta_proj, f, df_proj);
            end
            f = f_lastpos;
            df_proj = df_lastpos_proj;
            obj.theta = obj.theta_prior_step;
            obj.theta_proj = (obj.P.') * (obj.theta);
        end
    end  
end