/**
 * server.js
 * Copyright (c) 2000 - 2015 Samsung Electronics Co., Ltd. All rights reserved.
 *
 * Contact:
 * DongHee Yang <donghee.yang@samsung.com>
 * Sungmin Kim <sm.art.kim@samsung.com>
 * Jiil Hyoun <jiil.hyoun@samsung.com>
 * Jonghwan Park <iwin100.park@samsung.com>
 * Kitae Kim <kt920.kim@samsung.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Contributors:
 * - S-Core Co., Ltd
**/

var dibs = require('../../core/dibs');
var DError = require('../../core/exception');
var _ = require('underscore');
var async = require('async');
var JobSchedule = require('./job-schedule.js');
var utils = require('../../lib/utils');


/**
 * @module servers/job-manager/server
 */


/**
 * @function createServer
 * @param {string} serverId - serverId
 * @returns {string}
 * @memberOf module:servers/job-manager/server
 */

module.exports.createServer = function (serverId) {
    JobManagerServer.prototype = dibs.BaseServer.createServer(serverId, 'jobmgr');

    return new JobManagerServer();
};


function JobManagerServer() {
    this.name = 'job manager';

    var jobs = {}; // jobid  -> status, server
    var conflictJobs = {}; // working jobid  -> conflict jobid list
    var cancelReqJobs = {};
    var jobStatusListeners = {};
    var isSchedulingInitJob = false;
    var isSchedulingExecJob = false;
    var modifyStatusQueue = initJobStatusQueue();
    var scheduleOptions = {};
    var self = this;
    var oldConflictJobs = {};
    var oldAssigned = [];
    var recoveryJobQueue = {};

    // CALLBACK: for creating server's default configuration
    this.getDefaultConfiguration = function (baseConfig) {
        var defaultConfig = baseConfig;
        defaultConfig.schedule_using_exec_time = true;
        return defaultConfig;
    };

    this.OnServerStarting = function (callback) {
        callback(null);
    };

    this.OnServerStarted = function (callback) {
        // read configuration
        if (self.config.get('schedule_using_exec_time')) {
            scheduleOptions.scheduleUsingExecTime = true;
        }

        dibs.thisServer.addEventListener('SERVER_STATUS_CHANGED', { },
            function (evtObj, listenerCB) {
                switch (evtObj.status) {
                case 'DISCONNECTED':
                    afterWorkWhenBuilderDown(evtObj.server);
                    break;
                case 'RUNNING' :
                    recoveryJobsForRunnigServer(evtObj.server);
                    break;
                default :
                    break;
                }
                listenerCB(null);
            }, function (err) {
                if (err) {
                    self.log.error(err);
                }
            });

        callback(null);
    };

    this.OnMasterServerIsReady = function (callback) {
        recoveryJobs();

        // periodically call initialzing jog
        setInterval(function () {
            scheduleToInitializeJob();
        }, 1000);

        // call schedule & build periodically
        setInterval(function () {
            scheduleToExecuteJob();
        }, 1000);

        callback(null);
    };

    function afterWorkWhenBuilderDown(server) {
        var s = dibs.getServer(server.id);
        s.getEventListeners('JOB_STATUS_CHANGED', function (err, events) {
            if (!err) {
                _.each(events, function (e) {
                    var jobId = e.options.jobId;
                    var job = jobs[jobId];
                    if (job.status === 'PENDING') {
                        recoveryJobQueue[job.id] = _.clone(job);
                    } else {
                        job.error = new DError('JOBMGR009', {
                            serverId: server.id,
                            jobId: jobId
                        });
                        s.emitEvent({
                            event: 'JOB_STATUS_CHANGED',
                            status: 'ERROR',
                            jobId: jobId,
                            job: jobs[jobId]
                        });
                        if (!job.parentId) {
                            var builderServer = '';
                            if (job.execServerId) {
                                builderServer = job.execServerId + ' ' ;
                            } else if (job.initServerId) {
                                builderServer = job.initServerId + ' ' ;
                            } else {
                                builderServer = '';
                            }
                            // send error message to admin and owner
                            var msg = 'JOB ID: ' + jobId + '\n' +
                                'Distribution: ' + job.distName + '\n' ;
                            if (job.projectName) {
                                msg = msg + 'Project: ' + job.projectName + '\n';
                            } else {
                                if (job.subJobs && job.subJobs.length > 0) {
                                    msg = msg + 'Project: ' + _.uniq(_.map(job.subJobs, function (sub) {return sub.projectName;})).join(', ') + '\n';
                                }
                            }
                            msg = msg + 'Error: ' + job.id + ' was failed because of connection problem in server ' + builderServer + '\n\n';

                            sendEmail({
                                subject: '[JOB-' + jobId + ')] Build ' + job.status,
                                emailList: [job.userEmail],
                                groupList: ['administrator'],
                                message: msg
                            });
                        }
                    }
                });
            }
        });
    }

    function recoveryJobsForRunnigServer(server) {
        _.each(recoveryJobQueue, function (job) {
            if (job.execServerId === server.id) {
                pendingJobHandler(job, function (err) {
                    if (err) {
                        self.log.error(err);
                        // send error message to admin
                        var msg = 'JOB ID: ' + job.id + '\n' +
                            'Distribution: ' + job.distName + '\n' ;
                        if (job.projectName) {
                            msg = msg + 'Project: ' + job.projectName + '\n';
                        } else {
                            if (job.subJobs && job.subJobs.length > 0) {
                                msg = msg + 'Project: ' + _.uniq(_.map(job.subJobs, function (sub) {return sub.projectName;})).join(', ') + '\n';
                            }
                        }
                        msg = msg + 'Error: Recoverying ' + job.id + ' was failed because of the below problem\n\n' +
                            JSON.stringify(err);

                        sendEmail({
                            subject: '[JOB-' + job.id + ')] Recovery ' + job.status,
                            emailList: [],
                            groupList: ['administrator'],
                            message: msg
                        });
                    }
                    delete recoveryJobQueue[job.id];
                });
            }
        });
    }

    this.addJobInternal = function (userEmail, distName, prjName, environmentName, options, callback) {
        // create new job from prjName
        self.log.info('Adding new job... (' + distName + ',' + prjName + ',' + environmentName + ')');
        dibs.rpc.datamgr.addJob(userEmail, distName, prjName, environmentName, null, options, function (err, job) {
            if (err) {
                callback(err, null);
            } else {
                self.log.info('Added a new job #' + job.id);
                // add it into job queue
                jobs[job.id] = job;
                jobs[job.id].startTime = utils.getTimeString();
                setJobStatus(job.id, 'JUST_CREATED', null, function (err) {
                    callback(err, job.id);
                });
            }
        });
    };


    this.addSubJobInternal = function (parentId, prjName, environmentName, options, callback) {
        // create new job from prjName
        var parentJob = jobs[parentId];
        self.log.info('Adding new sub job ... (' + parentId + ',' + prjName + ')');
        dibs.rpc.datamgr.addJob(parentJob.userEmail, parentJob.distName,
            prjName, environmentName, parentId, options,
            function (err, job) {
                if (err) {
                    callback(err, null);
                } else {
                    self.log.info('Added a new job #' + job.id);
                    // add it into job queue
                    jobs[job.id] = job;
                    jobs[job.id].startTime = utils.getTimeString();
                    setJobStatus(job.id, 'JUST_CREATED', null, function (err) {
                        callback(err, job.id);
                    });
                }
            });
    };


    function cancelJobInternal(jobId, callback) {
        var error = null;
        // check exists
        if (jobs[jobId] === undefined) {
            error = new DError('JOBMGR001', {
                jobId: jobId
            });
            self.log.error(error.message);
            callback(error);
            return;
        }
        // check job status // JOB STATUS is CANCELING
        if (cancelReqJobs[jobId] !== undefined) {
            error = new DError('JOBMGR002', {
                jobId: jobId
            });
            self.log.error(error.message);
            callback(error);
            return;
        }
        if (jobs[jobId].status === 'FINISHED' ||
            jobs[jobId].status === 'ERROR' ||
            jobs[jobId].status === 'CANCELED') {
            error = new DError('JOBMGR004', {
                jobId: jobId
            });
            self.log.error(error.message);
            callback(error);
            return;
        }

        self.log.info('Received cancel-request of waiting job #' + jobId);
        // if assigned, get server that execute it
        // send cancel request
        if (jobs[jobId].status === 'WORKING' ||
            jobs[jobId].status === 'INITIALIZING' ||
            jobs[jobId].status === 'PENDING') {

            self.log.info('Canceling ' + jobs[jobId].status + ' job #' + jobId + '...');
            cancelReqJobs[jobId] = jobs[jobId];

            dibs.getServer(jobs[jobId].builderId).cancelJob(jobId, function (err) {
                if (err) {
                    self.log.error('Canceling job #' + jobId + ' failed with error!');
                    self.log.error(err);
                } else {
                    var subJobs = _.where(jobs, {parentId: jobId});
                    _.each(subJobs, function (subjob) {
                        cancelJobInternal(subjob.id, function (err) {
                            if (err) {
                                self.log.info(' Subjob of job #' + jobId + ' cancel is failed');
                                self.log.info(err.message);
                            } else {
                                self.log.info(' Subjob of job #' + jobId + ' is canceled by jobmanager');
                            }
                        });
                    });
                    self.log.info('Canceled working job #' + jobId);

                    async.each(jobs[jobId].subJobs,
                        function (subjob, cb) {
                            cancelJobInternal(subjob.id, cb);
                        }, function (err1) {
                            if (err1) {
                                self.log.info(err1.message);
                            } else {
                                self.log.info(' Subjob of job# ' + jobId + ' is canceled by jobmanager');
                            }
                        });
                }
                delete cancelReqJobs[jobId];
                callback(err);
            });
        } else if (jobs[jobId].status === 'INITIALIZED' && jobs[jobId].subJobs.length > 0) {
            self.log.info('Canceling ' + jobs[jobId].status + ' job #' + jobId + ' and it\'s subjobs ...');
            setJobStatus(jobId, 'CANCELED', null, callback);
            // remove from list
            if (jobs[jobId].parentId === null) {
                removeFromJobs(jobId);
            }

            self.log.info('Canceled job #' + jobId);
            async.each(jobs[jobId].subJobs,
                function (subjob, cb) {
                    cancelJobInternal(subjob.id, cb);
                },
                function (err) {
                    if (err) {
                        self.log.info(err.message);
                    } else {
                        self.log.info(' Subjob of job #' + jobId + ' is canceled by jobmanager');
                    }
                });
        } else { // JOB STATUS is INITIALIZED / JUST_CREATED
            setJobStatus(jobId, 'CANCELED', null, callback);

            // remove from list
            if (jobs[jobId].parentId === null) {
                removeFromJobs(jobId);
            }

            self.log.info('Canceled job #' + jobId);
            return;
        }

    }
    this.cancelJobInternal = cancelJobInternal;


    this.queryJobInternal = function (jobId, callback) {
        // check exists
        if (jobs[jobId] !== undefined) {
            callback(null, jobs[jobId]);
        } else {
            dibs.rpc.datamgr.searchJobs({
                id: jobId
            }, function (err, results) {
                if (err) {
                    callback(err, null);
                } else {
                    if (results.length > 0) {
                        callback(null, results[0]);
                    } else {
                        callback(new DError('JOBMGR001', {
                            jobId: jobId
                        }), null);
                    }
                }
            });
        }
    };


    function waitForJobStatusInternal(jobId, status, callback) {
        // check exists
        if (jobs[jobId]) {
            var stat = jobs[jobId].status;
            if (stat === status) {
                callback(null, status);
            } else if (stat === 'FINISHED' || stat === 'ERROR' || stat === 'CANCELED') {
                callback(new DError('JOBMGR005', {
                    jobId: jobId,
                    status: stat,
                    expected: status
                }), stat);
            } else {
                var listener = function (newStatus) {
                    if (newStatus === status) {
                        removeJobStatusChangeListener(jobId, listener);
                        callback(null, newStatus);
                    } else if (newStatus !== status && (newStatus === 'FINISHED' || newStatus === 'ERROR' || newStatus === 'CANCELED')) {
                        removeJobStatusChangeListener(jobId, listener);
                        callback(new DError('JOBMGR005', {
                            jobId: jobId,
                            status: newStatus,
                            expected: status
                        }), newStatus);
                    } else {
                        // do nothing
                    }
                };
                addJobStatusChangeListener(jobId, listener);
            }
        } else {
            dibs.rpc.datamgr.searchJobs({
                id: jobId
            }, function (err, results) {
                if (err) {
                    callback(err, null);
                } else if (results.length === 0) {
                    callback(new DError('JOBMGR001', {
                        jobId: jobId
                    }), null);
                } else {
                    if (results[0].status === status) {
                        callback(null, results[0].status);
                    } else {
                        callback(new DError('JOBMGR008', {
                            jobId: jobId,
                            status: results[0].status,
                            expected: status
                        }), results[0].status);
                    }
                }
            });
        }
    }
    this.waitForJobStatusInternal = waitForJobStatusInternal;

    this.waitForBuilderStatusInternal = function (builderId, callback) {
        var server = dibs.getServer(builderId);

        if (server) {
            waitForServerIsReady(builderId, callback);
        } else {
            callback(new DError('JOBMGR012', {builderId: builderId}));
        }
    };

    function waitForServerIsReady(sid, callback) {
        var retryCnt = 0;

        // Check server init
        var exec = setInterval(function () {
            var server = dibs.getServer(sid);
            retryCnt++;
            if (server.status === 'RUNNING') {
                clearInterval(exec);
                callback(null);
            } else {
                // wait for 5 minutes
                if (retryCnt > 300) {
                    clearInterval(exec);
                    callback(new DError('JOBMGR013'));
                }
            }
        }, 1000);
    }


    function initJobStatusQueue() {
        return async.queue(function (task, callback) {
            dibs.rpc.datamgr.updateJob(task.job, callback);
        }, 1);
    }


    function setJobStatus(jobId, status, error, callback) {
        dibs.log.info('Job #' + jobId + ' entered new status:"' + status + '"');

        var error_code = '';
        if ((status === 'ERROR' || status === 'CANCELED') && error) {
            error_code = error.errno;
            jobs[jobId].statusInfo = error.message;
        }

        if (_.contains(['ERROR', 'CANCELED', 'FINISHED'], status)) {
            jobs[jobId].endTime = utils.getTimeString();
        }

        jobs[jobId].status = status;
        jobs[jobId].error_code = error_code;
        jobs[jobId].board.push({
            type: 'STATUS',
            time: utils.getTimeString(),
            name: jobs[jobId].status
        });

        var clonedJob = _.clone(jobs[jobId]);

        // NOTE. this code prevents duplicated DB update of board info.
        jobs[jobId].board = [];

        modifyStatusQueue.push({
            job: clonedJob
        }, function (err) {
            // call change listner after db update
            executeJobStatusChangeListeners(jobId, status);

            // Update project statistics infomation
            if (jobs[jobId].projectName && (status === 'FINISHED' || status === 'CANCELED' || status === 'ERROR')) {
                updateProjectStatistics(jobId, callback);
            } else {
                if (callback) {
                    callback(null);
                }
            }

            if (err) {
                dibs.log.warn(err);
            }
        });

    }


    function addJobStatusChangeListener(jobId, listener) {
        if (jobStatusListeners[jobId] === undefined) {
            jobStatusListeners[jobId] = [];
        }
        jobStatusListeners[jobId].push(listener);
    }


    function removeJobStatusChangeListener(jobId, listener) {
        var listeners = jobStatusListeners[jobId];
        if (listeners !== undefined && listeners.indexOf(listener) !== -1) {
            listeners.splice(listeners.indexOf(listener), 1);
            if (listeners.length === 0) {
                delete jobStatusListeners[jobId];
            }
        }
    }


    function executeJobStatusChangeListeners(jobId, status) {
        var listeners = _.clone(jobStatusListeners[jobId]);
        if (listeners === undefined) {
            return;
        }
        for (var i = 0; i < listeners.length; i++) {
            listeners[i](status);
        }
    }


    function updateProjectStatistics(jobId, callback) {
        var job = null;

        async.waterfall([
            // search job
            function (cb) {
                dibs.rpc.datamgr.searchJobs({
                    id: jobId
                }, cb);
            },
            // search project
            function (results, cb) {
                job = results[0];

                // search project from DB
                dibs.rpc.datamgr.searchProjects({
                    name: job.projectName,
                    distName: job.distName
                }, cb);
            },
            // save project statistics
            function (projects, cb) {
                if (projects.length <= 0) {
                    return cb(new Error('No projects found!'));
                }

                var prj = projects[0];
                if (!prj.options.TOTAL_JOBS ||
                    prj.options.TOTAL_JOBS < 0) {
                    prj.options.TOTAL_JOBS = 0;
                }
                if (!prj.options.SUCCESS_JOBS ||
                    prj.options.SUCCESS_JOBS < 0) {
                    prj.options.SUCCESS_JOBS = 0;
                }
                if (!prj.options.ACC_SUCC_EXEC_TIME ||
                    prj.options.ACC_SUCC_EXEC_TIME < 0) {
                    prj.options.ACC_SUCC_EXEC_TIME = 0;
                }
                if (!prj.options.AVG_SUCC_EXEC_TIME ||
                    prj.options.AVG_SUCC_EXEC_TIME < 0) {
                    prj.options.AVG_SUCC_EXEC_TIME = 0;
                }

                prj.options.TOTAL_JOBS++;
                if (job.status === 'FINISHED') {
                    var initStartTimeObj = job.board.filter(function (e) {
                        return e.name === 'INITIALIZING';
                    })[0];
                    var initEndTimeObj = job.board.filter(function (e) {
                        return e.name === 'INITIALIZED';
                    })[0];
                    var workingStamps = job.board.filter(function (e) {
                        return e.name === 'WORKING';
                    });
                    var execEndTimeObj = job.board.filter(function (e) {
                        return e.name === 'FINISHED';
                    })[0];
                    if (initStartTimeObj && initEndTimeObj && workingStamps[0] && execEndTimeObj) {
                        // calculate exec time
                        var initStartTime = new Date(initStartTimeObj.time);
                        var initEndTime = new Date(initEndTimeObj.time);
                        var execEndTime = new Date(execEndTimeObj.time);
                        var execWorkingTime = null;
                        var execPendingTime = null;
                        // calculate exectime
                        var execTime = (initEndTime - initStartTime);
                        if (workingStamps.length > 1) {
                            var pendingStamps = job.board.filter(function (e) {
                                return e.name === 'PENDING';
                            });
                            for (var i = 0; i < workingStamps.length - 1; i++) {
                                execWorkingTime = new Date(workingStamps[i].time);
                                if (pendingStamps[i] && pendingStamps[i].time) {
                                    execPendingTime = new Date(pendingStamps[i].time);
                                } else {
                                    execPendingTime = execWorkingTime;
                                }
                                execTime = execTime + (execPendingTime - execWorkingTime);
                            }
                        }
                        execWorkingTime = new Date(job.board.filter(function (e) {
                            return e.name === 'WORKING';
                        })[workingStamps.length - 1].time);
                        execTime = execTime + (execEndTime - execWorkingTime);
                        prj.options.SUCCESS_JOBS++;
                        prj.options.ACC_SUCC_EXEC_TIME += execTime;
                        prj.options.AVG_SUCC_EXEC_TIME = Math.round((prj.options.ACC_SUCC_EXEC_TIME / 1000 / prj.options.SUCCESS_JOBS) * 10) / 10;
                    }
                }
                // update DB
                dibs.rpc.datamgr.updateProject(prj, function (err) {
                    // skip newProject
                    cb(err);
                });

            }
        ], function (err) {
            if (err) {
                if (job) {
                    self.log.warn('Updating project statistics failed...' + job.projectName + ':' + job.distName);
                } else {
                    self.log.warn('Updating project statistics failed...' + jobId);
                }
            } else {
                self.log.info('Updated project statistics...' + job.projectName + ':' + job.distName);
            }
            // even if it fails, don't care
            callback(null);
        });
    }


    /**
     * @function getRecommendedServer
     * @param {Array.object} builders - list of builder
     * @param {module:lib/utils.callback_error} callback
     * @memberOf module:servers/jobmanager/server
     */
    var getRecommendedServer = function (builders, callback) {
        var builderIds = _.map(builders, function (builder) {
            return builder.id;
        });
        async.map(builderIds, dibs.rpc.master.getServerStatus,
            function (err, statList) {
                if (err) {
                    callback(err, null);
                } else {
                    var cpuList = _.map(statList, function (stat) {
                        return stat.cpuUsage;
                    });
                    var minCpu = _.min(cpuList);
                    var index = _.indexOf(cpuList, minCpu);
                    callback(err, builders[index]);
                }
            });
    };


    //PRIVATE
    //

    // add job status change listener
    function addJobStatusChangeListenerToServer(jobId, builder, callback) {
        var jobListener = null;

        dibs.getServer(builder.id).addEventListener('JOB_STATUS_CHANGED', {
            jobId: Number(jobId)
        }, function (evtObj, cb) {
            var job = evtObj.job;
            var status = evtObj.status;

            if (!job || !job.id || (Number(job.id) !== Number(jobId))) {
                self.log.warn('Job #' + jobId + ' uses invalid job of event object: ');
                self.log.warn(evtObj);
                // TODO: send notificaton email to administrator.
                // executingevent-manager does nothing although cb returns null.
                return cb(null);
            }
            statusChangeHandler(job, status, builder, jobListener, cb);
        }, function (err, listener) {
            if (err) {
                self.log.error(err);
            }
            // set job status change listener
            jobListener = listener;

            callback(err, jobListener);
        });
    }

    function statusChangeHandler(job, status, builder, listener, callback) {
        switch (status) {
        case 'INITIALIZED':
            // update status of sub jobs
            jobs[job.id] = job;
            if (jobs[job.id].subJobs !== undefined) {
                _.each(jobs[job.id].subJobs, function (sub) {
                    jobs[sub.id] = sub;
                });
            }

            // update status
            setJobStatus(job.id, 'INITIALIZED', null, callback);

            // remove event listener
            if (listener) {
                dibs.getServer(builder.id).removeEventListener(listener,
                    function (err) {
                        if (err) {
                            self.log.warn(err);
                        }
                    });
            }

            self.log.info('The Job #' + job.id + ' is initialized successfully!');
            break;
        case 'FINISHED':
            // update status of sub jobs
            jobs[job.id] = job;
            if (jobs[job.id].subJobs !== undefined) {
                _.each(jobs[job.id].subJobs, function (sub) {
                    jobs[sub.id] = sub;
                });
            }

            // remove from mutually-exclusive-check-list
            if (conflictJobs[job.id]) {
                delete conflictJobs[job.id];
            }

            // update status
            setJobStatus(job.id, 'FINISHED', null, callback);
            messageGenerator('SUCCESS', jobs[job.id]);

            // remove from list
            if (jobs[job.id].parentId === null) {
                removeFromJobs(job.id);
            }

            // remove event listener
            if (listener) {
                dibs.getServer(builder.id).removeEventListener(listener,
                    function (err) {
                        if (err) {
                            self.log.warn(err);
                        }
                    });
            }

            self.log.info('The Job #' + job.id + ' is finished successfully!');
            break;
        case 'ERROR':
            // update status
            jobs[job.id] = job;

            // remove from mutually-exclusive-check-list
            if (conflictJobs[job.id]) {
                delete conflictJobs[job.id];
            }

            // update status
            setJobStatus(job.id, 'ERROR', job.error, callback);
            messageGenerator('FAIL', jobs[job.id]);

            // remove from list
            if (jobs[job.id].parentId === null) {
                removeFromJobs(job.id);
            }

            // remove event listener
            if (listener) {
                dibs.getServer(builder.id).removeEventListener(listener,
                    function (err) {
                        if (err) {
                            self.log.warn(err);
                        }
                    });
            }

            self.log.error('The Job #' + job.id + ' failed!');
            self.log.error(job.error);
            break;
        case 'CANCELED':
            // update status
            jobs[job.id] = job;

            // remove from mutually-exclusive-check-list
            if (conflictJobs[job.id]) {
                delete conflictJobs[job.id];
            }

            // update status
            setJobStatus(job.id, 'CANCELED', job.error, callback);

            // remove from list
            if (jobs[job.id].parentId === null) {
                removeFromJobs(job.id);
            }

            // remove event listener
            if (listener) {
                dibs.getServer(builder.id).removeEventListener(listener,
                    function (err) {
                        if (err) {
                            self.log.warn(err);
                        }
                    });
            }

            self.log.info('The Job #' + job.id + ' is canceled! :');
            break;
        case 'WORKING':
            // update status
            setJobStatus(job.id, 'WORKING', null, callback);

            self.log.info('The Job #' + job.id + ' is working now...');
            break;
        case 'PENDING':
            jobs[job.id] = job;

            self.log.info('The Job #' + job.id + ' is pending now');

            pendingJobHandler(job);
            setJobStatus(job.id, 'PENDING', null, callback);

            // remove event listener
            if (listener) {
                dibs.getServer(builder.id).removeEventListener(listener,
                    function (err) {
                        if (err) {
                            self.log.warn(err);
                        }
                    });
            }

            break;
        default:
            self.log.warn('The Job #' + job.id + ' status is invalid. Status: ' + status);
            callback(null);
            break;
        }
    }

    function pendingJobHandler(job, callback) {
        var projectType = dibs.projectTypes[job.projectType];

        if (projectType.module.pendingJobHandler) {
            projectType.module.pendingJobHandler(job, function (err, result) {
                if (err) {
                    self.log.error('The Job #' + job.id + ' is failed by pending job handler');
                    self.log.error(err);
                }

                self.log.info('The Job #' + job.id + "' is under resume process");
                var resumeOptions = {
                    error: err,
                    result: result
                };

                requestResumeJob(job, resumeOptions, function (err) {
                    if (err) {
                        self.log.error('The Job #' + job.id + ' is error in resumeJob');
                        self.log.error(err);
                    }

                    if (callback) {
                        callback(err);
                    }
                });
            });
        } else {
            self.log.error('The Job #' + job.id + ' is failed by system errror... Not found API - pendingJobHandler');
            setJobStatus(job.id, 'ERROR', 'Not found API - pendingJobHandler', callback);
            // remove from list
            if (jobs[job.id].parentId === null) {
                removeFromJobs(job.id);
            }
            if (callback) {
                callback(null);
            }
        }
    }

    function requestResumeJob(job, options, callback) {
        var cloneJob = _.clone(job);
        var builder = dibs.getServer(job.execServerId);
        if (!builder) {
            return callback(new DError('JOBMGR012', {builderId: job.execServerId}));
        }


        addJobStatusChangeListenerToServer(cloneJob.id, builder, function (err1, listener) {
            if (err1) {
                callback(err1);
                return;
            }

            self.log.info('The job #' + job.id + ' is resumed to server \'' + builder.id + '\'');
            builder.resumeJob(cloneJob, options, function (err2) {
                if (err2) {
                    self.log.error(err2);
                    // send error message to admin
                    var msg = 'JOB ID: ' + job.id + '\n' +
                        'Distribution: ' + job.distName + '\n' ;
                    if (job.projectName) {
                        msg = msg + 'Project: ' + job.projectName + '\n';
                    } else {
                        if (job.subJobs && job.subJobs.length > 0) {
                            msg = msg + 'Project: ' + _.uniq(_.map(job.subJobs, function (sub) {return sub.projectName;})).join(', ') + '\n';
                        }
                    }
                    msg = msg + 'Error: Resuming' + cloneJob.id + ' was failed because of the below problem\n\n' +
                        JSON.stringify(err2);

                    sendEmail({
                        subject: '[JOB-' + cloneJob.id + ')] Resume ' + cloneJob.status,
                        emailList: [],
                        groupList: ['administrator'],
                        message: msg
                    });
                    if (listener) {
                        builder.removeEventListener(listener, function (err3) {
                            if (err3) {
                                self.log.warn(err3);
                            }
                        });
                    }
                }
                callback(err2);
            });
        });
    }


    // request to build this job
    var requestToBuildJob = function (jobId, builder) {
        jobs[jobId].execServerId = builder.id;
        async.waterfall([
            function (cb) {
                addJobStatusChangeListenerToServer(jobId, builder, cb);
            },
            function (listener, cb) {
                setJobStatus(jobId, 'WORKING', null, function () { });
                builder.buildJob(jobs[jobId], function (err) {
                    if (err) {
                        self.log.error(err);
                    }
                    if (err && listener) {
                        dibs.getServer(builder.id).removeEventListener(listener,
                            function (err) {
                                if (err) {
                                    self.log.warn(err);
                                }
                            });
                    }
                    cb(err);
                });
            }
        ], function (err) {
            if (err) {
                self.log.warn(err);
            }
        });
    };


    function removeFromJobs(jobId) {
        if (jobs[jobId]) {
            setTimeout(function () {
                if (jobs[jobId].subJobs) {
                    _.each(jobs[jobId].subJobs, function (sub) {
                        delete jobs[sub.id];
                    });
                }
                delete jobs[jobId];
                self.log.info('** Jobs: ' + getJobInfo().toString());
            }, 60000);
        }
    }


    // request to initialize this job
    function requestToInitializeJob(jobId, builder) {
        jobs[jobId].initServerId = builder.id;
        async.waterfall([
            function (cb) {
                addJobStatusChangeListenerToServer(jobId, builder, cb);
            },
            function (listener, cb) {
                setJobStatus(jobId, 'INITIALIZING', null, function () { });
                builder.initializeJob(jobs[jobId], function (err) {
                    if (err) {
                        self.log.error(err);
                    }

                    if (err && listener) {
                        dibs.getServer(builder.id).removeEventListener(listener, function (err) {
                            if (err) {
                                self.log.warn(err);
                            }
                        });
                    }
                    cb(err);
                });
            }
        ], function (err) {
            if (err) {
                self.log.warn(err);
            }
        });
    }


    function scheduleToInitializeJob() {
        if (isSchedulingInitJob) {
            return;
        }
        isSchedulingInitJob = true;

        JobSchedule.scheduleToInitializeJobs(jobs, function (err, result) {
            if (!err) {
                for (var jobId in result.assignedJobs) {
                    if (!jobs[jobId].initServerId) {
                        var builder = result.assignedJobs[jobId];
                        jobs[jobId].builderId = builder.id;
                        self.log.info('The job #' + jobId + ' is assigned to server \'' + builder.id + '\'');
                        requestToInitializeJob(jobId, builder);
                    }
                }
            }
            isSchedulingInitJob = false;
        });
    }
    function keys(obj) {
        var list = [];
        for (var key in obj) {
            list.push(key);
        }
        return list;
    }

    function getJobInfo() {
        return _.map(_.keys(jobs), function (id) {
            return '#' + id + '[' + jobs[id]['status'] + ']';
        });
    }

    function scheduleToExecuteJob() {
        if (isSchedulingExecJob) {
            return;
        }
        isSchedulingExecJob = true;

        // update builders
        JobSchedule.scheduleToBuildJobs(jobs, conflictJobs, scheduleOptions, function (err, result) {
            var assignedJobsKeyList = _.keys(result.assignedJobs);

            if (!_.isEqual(conflictJobs, oldConflictJobs) || !_.isEqual(assignedJobsKeyList, oldAssigned)) {
                self.log.info('** Jobs: ' + getJobInfo().toString());
                self.log.info('** Conflicts: ' + JSON.stringify(conflictJobs));
                self.log.info('** Assgined: ' + JSON.stringify(assignedJobsKeyList));
                oldConflictJobs = _.clone(conflictJobs);
                oldAssigned = _.clone(assignedJobsKeyList);
            }

            if (!err) {
                for (var jobId in result.assignedJobs) {
                    if (!jobs[jobId].execServerId) {
                        var builder = result.assignedJobs[ jobId ];
                        jobs[jobId].builderId = builder.id;

                        // add entry to mutual exclusive check list
                        if (!jobs[jobId].parentId) {
                            conflictJobs[jobId] = [];
                        }
                        self.log.info('The job #' + jobId + ' is assigned to server \'' + builder.id + '\'');
                        requestToBuildJob(jobId, builder);
                    }
                }
            }
            isSchedulingExecJob = false;
        });
    }

    function messageGenerator(trigger, job) {
        if (!dibs.getServersByType('messenger')[0]) {
            self.log.info('messenger server is not exists');
            self.log.info('ignore sending email');
            return;
        }

        // write message
        // var subject = job.projectName + ' - Job No.' + job.id + ' Build ' + job.status;
        var subject = '[JOB-' + job.id + '] Build ' + job.status;
        var msg = 'Started by ' + job.userEmail + '\n\n' +
            '<< Information >>' + '\n' +
            '- Distribution: ' + job.distName + '\n' +
            '- Project Name: ' + job.projectName + '\n' +
            '- Environment Name: ' + job.environmentName + '\n' +
            '- Build Number: ' + job.id + '\n' +
            '- Build Time: ' + job.startTime + ' ~ ' + job.endTimei + '\n';

        // add error msg
        if (trigger === 'FAIL') {
            msg += '- Error Code: ' + job.error_code + '\n' +
                   '- Error Log: ' + job.statusInfo + '\n';
        }

        var nList = job.options.NOTIFICATIONS;
        nList.forEach(function (notification) {
            if (notification.event === trigger) {
                var method = notification.method;
                var targets = notification.targetData;
                dibs.rpc.messenger.notify(method, subject, targets, msg, function (err) {
                    if (err) {
                        self.log.info('sending email failed');
                        self.log.info(err);
                    }
                    self.log.info('sending email succeeded');
                });
            }
        });
    }


    function recoveryJobs() {
        async.waterfall([
            function (cb1) {
                self.log.info('Start recovery jobs...');
                dibs.rpc.datamgr.searchJobs({
                    status: ['JUST_CREATED', 'INITIALIZING', 'INITIALIZED', 'WORKING', 'PENDING']
                }, function (err, recoveryJobList) {
                    if (err) {
                        self.log.error(err);
                        cb1(err);
                    } else {
                        cb1(null, recoveryJobList);
                    }
                });
            },
            function (recoveryJobList, cb1) {
                async.each(recoveryJobList,
                    function (job, cb2) {
                        jobs[job.id] = job;

                        if (_.contains(['JUST_CREATED', 'INITIALIZING', 'INITIALIZED', 'WORKING'], job.status)) {
                            self.log.warn('Correcting job status: ' + job.id + ' ' + job.status + ' => CANCELED');
                            var error = new DError('JOBMGR011', {
                                jobId: job.id
                            });
                            setJobStatus(job.id, 'CANCELED', error, function (err) {
                                if (err) {
                                    self.log.error(err);
                                } else {
                                    self.log.warn('Remove correcting job(' + job.id + ') from JOBS');
                                    delete jobs[job.id];
                                }
                                cb2(null);
                            });
                            if (!job.parentId) {
                                var builderServer = '';
                                if (job.execServerId) {
                                    builderServer = job.execServerId + ' ' ;
                                } else if (job.initServerId) {
                                    builderServer = job.initServerId + ' ' ;
                                } else {
                                    builderServer = '';
                                }

                                var msg = 'JOB ID: ' + job.id + '\n' +
                                    'Distribution: ' + job.distName + '\n' ;
                                if (job.projectName) {
                                    msg = msg + 'Project: ' + job.projectName + '\n';
                                } else {
                                    if (job.subJobs && job.subJobs.length > 0) {
                                        msg = msg + 'Project: ' + _.uniq(_.map(job.subJobs, function (sub) {return sub.projectName;})).join(', ') + '\n';
                                    }
                                }
                                msg = msg + 'Error: ' + job.id + ' was canceled because of connection problem in server ' + builderServer + '\n\n';

                                sendEmail({
                                    subject: '[JOB-' + job.id + ')] Build ' + job.status,
                                    emailList: [job.userEmail],
                                    groupList: ['administrator'],
                                    message: msg
                                });
                            }
                        } else if (job.status === 'PENDING') {
                            recoveryJobQueue[job.id] = job;
                            //for dependency checking
                            conflictJobs[job.id] = [];
                            cb2(null);
                        } else {
                            delete jobs[job.id];
                            cb2(new DError('JOBMGR010', {
                                status: job.status
                            }));
                        }
                    },
                    function (err) {
                        cb1(err);
                    });
            },
            function (cb) {
                var builders = dibs.getServersByType('builder');
                async.each(builders, function (builder, cb1) {
                    if (builder.status === 'RUNNING') {
                        recoveryJobsForRunnigServer(builder);
                    }
                    cb1(null);
                }, function (err) {
                    cb(err);
                });
            }
        ],
        function (err) {
            if (!err) {
                self.log.info('Finish load recovery jobs...');
            } else {
                self.log.Error('recovery jobs start failed');
                self.log.Error(err);
            }
        });
    }

    // email contents:
    //      subject: string
    //      emailList: array
    //      groupList: array
    //      message: string
    function sendEmail(contents) {
        if (!contents.groupList) {
            contents.groupList = [];
        }
        if (!contents.emailList) {
            contents.emailList = [];
        }
        if (dibs.getServersByType('messenger')[0]) {
            async.map(contents.groupList, dibs.rpc.datamgr.searchUsersByGroupName,
                    function (err, result) {
                        var groupEmailList = [];
                        if (!err) {
                            groupEmailList = _.map(_.flatten(result), function (user) {
                                return user.email;
                            });
                        }
                        dibs.rpc.messenger.notify('email', contents.subject, _.difference(_.union(groupEmailList, contents.emailList), ['admin@user', 'sync-manager@user']),
                            contents.message, function (err) {
                                if (err) {
                                    self.log.info(err);
                                }
                            });
                    });
        }
    }
}
