Ashan take a look on #2233

If this is the latest version of jQuery form validator instead of file which is already in master branch,
then you could use a jQuery style code for isValid method

            // Run custom form validators if present
            $.each(custom, function(key, validator) {
                    if (validator.exec() !== true) {
                            valid = false;
                    }
            });

also you can simplify radio and checkbox validation in method: validate
#2294

                    if (tagName === 'fieldset' && ($el.hasClass('radio') || $el.hasClass('checkboxes'))) {
                            if (!$el.find('input:checked').length)
                                    handleResponse(false, $el);
                                    return false;
                             }
                    }

@piotrmocko
Latest commits includes your modifications as well as Achal's

@Achal-Aggarwal I have included your modifications to this PR by referring to #2233

I think that you can not cache form fields in this way, because if you will have two forms on your site, then first form which would run this method would be cached in $formFields. Also you do not have to look for buttons in this method because it has nothing to validate.

isValid = function(form) {
    var valid = true, $form = $(form), i, message, errors, error, label;
    // Validate form fields
    $formFields = $formFields || $form.find('input, textarea, select, button, fieldset');
    ...

Also you can add one more improvement to speed up labels search. Look for label first by ID, because in Joomla labels have the same ID as field but with suffix -lbl, and if not found then look for by REF attribute but only in form children and not in whole DOM.

To sum up here are my changes.

Objects to cache forms and forms fields.

var JFormValidator = function() {
    var $, handlers, inputEmail, custom, forms, formsFields, labels,
    ...

Init cache objects and cache form element by its name. Changed labels type from [] to {}.

initialize = function() {
    ...
    labels = {};
    forms = {};
    formsFields = {};
    ...
    // Attach to forms with class 'form-validate'
    $('form.form-validate').each(function(i) {
        var name = $(this).attr('name');
        if (!name) {
            name = 'jform-validate-'+i;
            $(this).attr('name', name);
        }
        forms[name] = $(this);
        attachToForm(this);
    }, this);

Cache form fields in formsFields[formName] and store form name in each field

attachToForm = function(form) {
    // Cache form fields
    var formName = $(form).attr('name');
    formsFields[formName] = [];
    // Iterate through the form object and attach the validate method to all input fields.
    $(form).find('input,textarea,select,fieldset,button').each(function() {
        ...
        if ((tagName === 'input' || tagName === 'button') && $el.attr('type') === 'submit') {
            ...
        } else {
            if (tagName !== 'fieldset') {
                $el.on('blur', function() {
                    return validate(this);
                });
                if ($el.hasClass('validate-email') && inputEmail) {
                    $el.get(0).type = 'email';
                }
            }
            $el.data('jform-name', formName);
            formsFields[formName].push($el);
        }
    });
},

Only go through cached fields

isValid = function(form) {
    var valid = true, i, message, errors, error, label;
    // Validate form fields
    $.each(formsFields[$(form).attr('name')], function(index, el) {
    ...

Find label by ID and then by ref in form

findInputLabel = function($el){
    var id = $el.attr('id'), label;
    if(!id){
        return false;
    }else if(typeof labels[id] !== 'undefined'){
        label = labels[id];
    }else{
        label = $('#'+ id +'-lbl');
        if (!label.length) {
            label = forms[$el.data('jform-name')].find('label[for="'+ id +'"]');
            if (!label.length)
                return false;
        }
        labels[id] = label;
    }
    return label;
},

I think that there are better solution to cache form and its fields then above.
You could write jQuery plugin to validate form and store reference to this plugin in data attribute in

I have done the following improvements for the validation.js also considering the above mentioned suggestions.

• Store form input fields references in form's own jQuery data object and store input fields label references within input field's own jQuery data object

 attachToForm = function(form) {
        var inputFields = [];
        // Iterate through the form object and attach the validate method to all input fields.
        $(form).find('input, textarea, select, button').each(function() {
            var $el = $(this), id = $el.attr('id'), tagName = $el.prop("tagName").toLowerCase();
            if ($el.hasClass('required')) {
                $el.attr('aria-required', 'true').attr('required', 'required');
            }
            if ((tagName === 'input' || tagName === 'button') && $el.attr('type') === 'submit')
            {
                if ($el.hasClass('validate')) {
                    $el.on('click', function() {
                        return isValid(form);
                    });
                }
            } else {
                if (tagName !== 'fieldset') {
                    $el.on('blur', function() {
                        return validate(this);
                    });
                    if ($el.hasClass('validate-email') && inputEmail) {
                        $el.get(0).type = 'email';
                    }
                }
                $el.data('label', findLabel(id, form)); // Store label reference
                inputFields.push($el);
            }
        });    
        $(form).data('inputfields', inputFields); // Store input field references
    },

This helps to avoid any memory leaks that can happen if a DOM element is removed, since jQuery also removes DOM element's data object.

• When validating form fields in isValid function, it uses the form's data object to iterate input fields reducing search overhead

   // Validate form fields
   isValid = function(form) {
        var valid = true, i, message, errors, error, label;
        $.each($(form).data('inputfields'), function(index, el) {
           if (validate(el) === false) {
              valid = false;
           }
       });
     // more...
    }

• Improving label search

 findLabel = function(id, form){
        var $label, $form = $(form);
        if (!id) {
            return false;
        }
        $label = $form.find('#' + id + '-lbl');
        if ($label.length) {
            return $label;
        }
        $label = $form.find('label[for="' + id + '"]');
        if ($label.length) {
            return $label;
        }
        return false;
  }

  handleResponse = function(state, $el) {
        var $label = $el.data('label');
        // more..
  }

You have lost radio and checkboxes validation. Add fieldset in following part of code:

    attachToForm = function(form) {
            ...
            $(form).find('input, textarea, select, fieldset, button').each(function() {

Two years ago I have proposed changes to Mootools validator, but it was never added to Joomla:
http://joomlacode.org/gf/project/joomla/tracker/?action=TrackerItemEdit&tracker_id=8103&tracker_item_id=26114
And it has on blur validation for radio and checkbox, but it is not the best and fastest way.

Here is improved code which has on blur validation for radio and checkbox.
Also it would validate single checkbox with required attribute like input text field without any fieldset.
Also it prevents from adding any button to validation queue.

attachToForm = function(form) {
    var inputFields = [];
    // Iterate through the form object and attach the validate method to all input fields.
    $(form).find('input, textarea, select, fieldset, button').each(function() {
        var $el = $(this), id = $el.attr('id'), tagName = $el.prop("tagName").toLowerCase(), tagType = $el.attr('type'), attach;
        if ($el.hasClass('required')) {
            $el.attr('aria-required', 'true').attr('required', 'required');
        }
        if ((tagName === 'input' && (tagType === 'button' || tagType === 'submit')) || tagName === 'button') {
            if ($el.hasClass('validate') && tagType === 'submit') {
                $el.on('click', function() {
                    return isValid(form);
                });
            }
        } else {
            attach = true;
            if (tagName !== 'fieldset') {
                if (tagName === 'input' && (tagType === 'radio' || tagType === 'checkbox')) {
                    var $fieldset = $el.closest('fieldset');
                    if ($fieldset.length) {
                        $el.data('fieldset', $fieldset);
                        $el.on('blur', function() {
                            return validate($(this).data('fieldset'));
                        });
                        attach = false;
                    } 
                    else {
                        $el.on('blur', function() {
                            return validate(this);
                        });
                    }
                }
                else {
                    $el.on('blur', function() {
                        return validate(this);
                    });
                    if ($el.hasClass('validate-email') && inputEmail) {
                        $el.get(0).type = 'email';
                    }
                }
            }
            if (attach) {
                $el.data('label', findLabel(id, form));
                inputFields.push($el);
            }
        }
    }); 
    $(form).data('inputfields', inputFields);
},

You can consider to skip also input hidden fields.

    } else {
        attach = true;
        if (tagName === 'input' && tagType === 'hidden') {
            attach = false;
        }
        else if (tagName !== 'fieldset') {
            if (tagName === 'input' && (tagType === 'radio' || tagType === 'checkbox')) {

I have included the missing radio and checkboxes validation.

Merged to 3.3-dev