/**
 * server.js
 * Copyright (c) 2000 - 2017 Samsung Electronics Co., Ltd. All rights reserved.
 *
 * Contact:
 * Sungmin Kim <sm.art.kim@samsung.com>
 * Jonghwan Park <iwin100.park@samsung.com>
 * Kitae Kim <kt920.kim@samsung.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Contributors:
 * - S-Core Co., Ltd
**/

var async = require('async');
var path = require('path');
var spawn = require('child_process').spawn;
var fs = require('fs');
var extfs = require('fs-extra');
var os = require('os');
var _ = require('underscore');

var dibs = require('../../core/dibs.js');
var dfs = require('../dibs.dist-fs/dist-fs.js');
var utils = require('../../lib/utils.js');
var DError = require('../../core/exception.js');
var DHandle = require('../../core/error-handler.js');
var Builder = require('../dibs.core.server/pkg-builder.js');
var Installer = require('../dibs.core.server/pkg-installer.js');
var FileSystem = require('../dibs.core/filesystem.js');
var Monitor = require('../../lib/monitor.js');


module.exports.createServer = function (serverId) {
    MasterServer.prototype = dibs.BaseServer.createServer(serverId, 'master');

    return new MasterServer();
};

function MasterServer() {
    var self = this;
    var DEFAULT_AGNET_PORT = 38971;

    this.name = 'master';

    // For managing server lifecycle
    var activeServers = {};
    var disconnectedTimes = {};
    var DISKSPACE_LIMIT = 80;
    var NOTIFICATION_DELAY = (1000 * 60 * 60 * 4); // 4 Hour

    // CALLBACK: for creating server's default configuration
    this.getDefaultConfiguration = function (baseConfig) {
        var defaultConfig = baseConfig;
        defaultConfig['dfs-size'] = 1024 * 1024 * 1024 * 10; // 10G
        return defaultConfig;
    };

    this.OnServerStarting = function (callback) {
        if (isLaunchedUsingSourceCode()) {
            dibs.log.info('Master server is launched using source code!');
            if (self.startOptions.install) {
                dibs.log.info('Master server is launched using \'--install\' option!');
                console.log('Installing DIBS...');
                var installDir = self.startOptions.install.length > 0 ?
                    path.resolve(self.startOptions.install) :
                    path.join(__dirname, '..', '..');
                installAndforkNewServer(installDir, callback);
            } else {
                callback(null);
            }
        } else {
            if (existsUpdates()) {
                dibs.log.info('Updates exists!');
                console.log('Updates exists! Updating DIBS...');
                if (isLaunchedUsingUpdateMode()) {
                    dibs.log.info('Master server is launched using \'--update\' option!');
                    updateAndforkNewServer(callback);
                } else {
                    forkNewServerForUpdate(callback);
                }
            } else {
                // prepare for updates
                var updatesDir = path.join(__dirname, '..', '..', 'updates');
                if (!fs.existsSync(updatesDir)) {
                    extfs.mkdirsSync(updatesDir);
                }

                callback(null);
            }
        }
    };


    this.OnServerStarted = function (callback) {
        dibs.config.createServerConfig(self, this.config.getConfigData());
        // Add child servers
        addServers(this.config.get('sub_servers'));

        var servers = dibs.getAllServers();

        async.waterfall([
            function (cb) {
                dibs.log.info('Opening DFS repository...');
                dfs.openRepository(self.config.get('dfs-path'),
                    self.config.get('dfs-size'),
                    self.startOptions.cleanStart,
                    self.port + 1,
                    function (err) {
                        if (!err) {
                            dibs.log.info('DFS repository loaded!');
                        }
                        cb(err);
                    });
            },
            function (cb) {
                self.addConnectionHandler('dfs', dfs.server);
                dfs.connect(cb);
            },
            function (cb) {
                console.log(' << attempting to launch each server >>');

                var subServers = _.filter(servers, function (server) {
                    return (server.type !== 'master');
                });

                var localBuildServers = [];
                var localNonBuildServers = [];
                var remoteBuildServers = [];
                var remoteNonBuildServers = [];
                _.each(subServers, function (server) {
                    if (server.name === 'builder') {
                        if (utils.isLocalAddr(server.host)) {
                            localBuildServers.push(server);
                        } else {
                            remoteBuildServers.push(server);
                        }
                    } else {
                        if (utils.isLocalAddr(server.host)) {
                            localNonBuildServers.push(server);
                        } else {
                            remoteNonBuildServers.push(server);
                        }
                    }
                });

                async.parallel({
                    local: async.apply(startLocalServers, localNonBuildServers),
                    localBuilder: async.apply(startLocalServersIgnoreBuilderError, localBuildServers),
                    remote: async.apply(startRemoteServers, remoteNonBuildServers),
                    remoteBuilder: async.apply(startRemoteServersIgnoreBuilderError, remoteBuildServers)
                },
                    function (err, results) {
                        var localErrors = results.local.filter(function (e) {
                            return e;
                        });

                        var remoteErrors = _.flatten(results.remote).filter(function (e) {
                            return e;
                        });

                        if ((localErrors.length > 0) || (remoteErrors.length > 0)) {
                            if (localErrors.length > 0) {
                                console.log('< local server Errors : >');
                                console.log(localErrors);
                            }
                            if (remoteErrors.length > 0) {
                                console.log('< remote server Errors : >');
                                console.log(remoteErrors);
                            }
                            cb(new DError('MASTER013'));
                        } else {
                            console.log(' << end >>');
                            cb(err);
                        }
                    });
            },
            function (cb) {
                initiateToMonitorServers(servers);

                // add servers to active server list
                for (var i = 0; i < servers.length; i++) {
                    activeServers[servers[i].id] = servers[i];
                }

                waitForCoreServersAreReady(cb);
            },
            function (cb) {
                if (dibs.getServersByType('datamgr').length > 0) {
                    syncWithDB(cb);
                } else {
                    cb(null);
                }
            }
        ], function (err) {
            if (err) {
                self.terminate({}, function () {
                    callback(err);
                });
            } else {
                callback(null);
            }
        });
    };


    this.OnMasterServerIsReady = function (callback) {
        callback(null);
    };


    function syncWithDB(callback) {
        // 1. get config from db
        getConfigFromDB({}, function (err, dbConfigs) {
            if (err) {
                console.log(err);
                callback(err);
            } else if (dbConfigs.length === 0) {
                //migrate to DB
                async.waterfall([
                    //1. wait all servers are ready
                    async.apply(waitForAllServersAreReady),
                    function (cb) {
                        var allServers = dibs.getAllServers();
                        var servers = _.filter(allServers, function (svr) {
                            return svr.id !== self.id;
                        });
                        setConfigFromRealServers(servers, cb);
                    },
                    function (cb) {
                        //2. push all configs to datamgr
                        var servers = dibs.getAllServers();
                        var hostIps = [];
                        _.each(servers, function (server) {
                            if (hostIps.indexOf(server.host) < 0) {
                                hostIps.push(server.host);
                            }
                        });
                        async.eachSeries(hostIps, function (host, cb1) {
                            self.log.info('Add ' + host + ' hosts to DB');
                            dibs.rpc.datamgr.addServer(host, 'host', null, null, host, null, null, null, null, cb1);
                        }, function (err) {
                            cb(err);
                        });
                    },
                    async.apply(migrateConfigs)
                ],
                    function (err) {
                        if (err) {
                            console.log(err);
                        } else {
                            self.log.info('Finished After Works');
                        }
                        callback(err);
                    });
            } else {
                async.waterfall([
                    //1. wait all servers are ready
                    async.apply(applyConfigFromDB, dbConfigs)
                ],
                    function (err) {
                        if (err) {
                            console.log(err);
                        } else {
                            self.log.info('Finished After Works');
                        }

                        callback(err);
                    });

            }
        });
    }

    this.OnServerTerminating = function (callback) {
        var servers = dibs.getAllServers();
        async.eachSeries(servers,
            function (s, cb) {
                if ((s.id !== self.id) &&
                    (s.status === 'INITIALIZED' || s.status === 'RUNNING')) {

                    self.log.info('Terminating sub-server... ' + s.id);
                    // NOTE. console out invokes server down
                    //console.log( "Terminating sub-server... " + s.id );
                    s.terminateServer(function (err) {
                        if (err) {
                            self.log.info('Error while terminating sub-server... ' +
                                s.id + '=>' + err.message);
                        }
                        cb(null);
                    });
                } else {
                    cb(null);
                }
            },
            function (err) {
                // close dfs repo
                if (dfs.isOpened()) {
                    dfs.closeRepository();
                }

                callback(err);
            });
    };


    this.innerGetServerStatus = function (sid, callback) {
        if (sid === this.id) {
            this.getStatus(function (err, serverStatus) {
                if (err) {
                    console.error('failed to get server status from ' + sid);
                }
                callback(err, serverStatus);
            });
        } else {
            var s = dibs.getServer(sid);
            if (s !== null) {
                s.getStatus(function (err, serverStatus) {
                    if (err) {
                        console.error('failed to get server status from ' + sid);
                    }
                    callback(err, serverStatus);
                });
            } else {
                callback(null, {
                    status: 'DISCONNECTED',
                    cpuUsage: 0,
                    memUsage: 0,
                    diskUsage: 0
                }
                );
            }
        }
    };


    this.startServerInternal = function (sid, callback) {
        var servers = dibs.getAllServers().filter(function (e) {
            return e.id === sid;
        });

        if (servers.length === 0) {
            callback(new DError('MASTER001', {
                id: sid
            }));
            return;
        }

        var server = servers[0];
        if (server.type === 'master') {
            callback(new DError('MASTER002'));
            return;
        }

        async.series([
            function (cb) {
                self.log.info('Launching sub-server... ' + server.id);
                launchServer(server, cb);
            },
            function (cb) {
                // set active
                activeServers[server.id] = server;

                // wait for ready
                waitForServerIsReady(sid, cb);
            }
        ], function (err) {
            callback(err);
        });
    };


    this.stopServerInternal = function (sid, callback) {
        var server = dibs.getServer(sid);
        if (!server) {
            callback(new DError('MASTER001', {
                id: sid
            }));
            return;
        }

        if (server.type === 'master') {
            dibs.log.info('Making master server stopped...');
            stopMyself(callback);
        } else {
            dibs.log.info('Making sub server stopped...' + sid);
            stopSubServer(server, function (err) {
                // Remove from active servers
                if (!err) {
                    delete activeServers[server.id];
                }

                callback(err);
            });
        }
    };

    this.createServerConfig = function (server, config) {
        dibs.config.createServerConfig(server, config);
    };

    this.restartServerInternal = function (sid, callback) {

        var server = dibs.getServer(sid);
        if (!server) {
            callback(new DError('MASTER001', {
                id: sid
            }));
            return;
        }

        if (server.type === 'master') {
            dibs.log.info('Restarting master server...');
            relaunchMyself(callback);
        } else {
            dibs.log.info('Restarting sub server...' + sid);
            relaunchSubServer(server, function (err) {
                if (!err) {
                    waitForServerIsReady(sid, callback);
                } else {
                    callback(err);
                }
            });
        }
    };


    this.downloadUpdateInternal = function (pkgName, options, callback) {
        if (options.remoteURL) {
            if (!options.distName) {
                callback(new DError('MASTER015')); return;
            }

            callback(new Error('Not supported yet!'));
        } else {
            dibs.log.info('Downloading update from internal repository...');
            downloadUpdateFromInternalRepository(pkgName, options, callback);
        }
    };


    this.checkUpdatesInternal = function (options, callback) {
        if (options.remoteURL) {
            if (!options.distName) {
                callback(new DError('MASTER015')); return;
            }

            callback(new Error('Not supported yet!'), '0.0.0', []);
        } else {
            checkUpdatesFromInternalRepository(options, callback);
        }
    };

    this.addServerInternal = function (serverType, serverInfo, config, callback) {
        self.log.info('add New server');
        self.log.info(serverInfo);
        self.log.info(config);
        if (!serverInfo.id || !_.isString(serverInfo.id)) {
            return callback(new DError('MASTER001', {
                id: serverInfo.id
            }));
        }

        if (!serverInfo.host || !_.isString(serverInfo.host)) {
            return callback(new DError('MASTER018', {
                host: serverInfo.host
            }));
        }

        if (!serverInfo.port || !serverInfo.port.toString().match(/^[0-9]*$/)) {
            return callback(new DError('MASTER019', {
                port: serverInfo.port
            }));
        }

        if (!serverInfo.type || serverInfo.type !== serverType) {
            serverInfo.type = serverType;
        }

        if (dibs.getServer(serverInfo.id)) {
            callback(new Error(serverInfo.id + ' server already exist'));
        } else {
            var serverConfig = {};
            addServer(serverType, serverInfo);
            var newServer = dibs.getServer(serverInfo.id);
            newServer.id = serverInfo.id;

            async.waterfall([
                function (cb) {
                    if (utils.isLocalAddr(newServer.host)) {
                        cb(null, dibs.config.getBaseConfig({
                            id: newServer.id,
                            host: newServer.host,
                            port: newServer.port,
                            type: serverType
                        }));
                    } else {
                        var handler = {
                            try: function () {
                                newServer.agentServer.getBaseConfig({
                                    id: newServer.id,
                                    host: newServer.host,
                                    port: newServer.port,
                                    type: serverType
                                }, cb);
                            },
                            catch: function (err) {
                                self.log.error('get base config using Agent Failed : ' + newServer.id);
                                self.log.error(err);
                                cb(err);
                            }
                        };
                        DHandle.run(handler);
                    }
                },
                function (baseConfig, cb) {
                    self.log.info(' - set config to DB');
                    var defaultConfig = newServer.getDefaultConfiguration(baseConfig);
                    var newConf = config;
                    newConf = _.omit(newConf, 'id');
                    newConf = _.omit(newConf, 'host');
                    newConf = _.omit(newConf, 'port');
                    newConf = _.omit(newConf, 'type');
                    if (newConf && (typeof newConf) === 'object' && newConf !== {}) {
                        _.each(newConf, function (val, key) {
                            defaultConfig[key] = val;
                        });
                    }
                    serverConfig = defaultConfig;
                    newServer.config = dibs.config.newServerConfig(newServer, serverConfig);
                    setConfigToDB(serverConfig, function (err) {
                        cb(err);
                    });
                },
                function (cb) {
                    if (utils.isLocalAddr(newServer.host)) {
                        self.log.info(' - generate local server config ' + newServer.id);
                        self.createServerConfig(newServer, serverConfig);
                        cb(null);
                    } else {
                        self.log.info(' - generate remote server config ' + newServer.id);
                        self.createRemoteServerConfig(newServer, serverConfig, cb);
                    }
                },
                function (cb) {
                    if (utils.isLocalAddr(newServer.host)) {
                        self.log.info(' - start local server ' + newServer.id);
                        startLocalServers([newServer], function (err, error) {
                            if (!err) {
                                // err must be null
                            }
                            cb(_.flatten(error)[0]);
                        });
                    } else {
                        self.log.info(' - start remote server ' + newServer.id);
                        startRemoteServers([newServer], function (err, error) {
                            if (!err) {
                                // err must be null
                            }
                            cb(_.flatten(error)[0]);
                        });
                    }
                },
                function (cb) {
                    monitorSubServer(newServer.id);
                    activeServers[newServer.id] = newServer;
                    monitorServerResourceUsage(newServer.id);
                    waitForServerIsReady(newServer.id, cb);
                }
            ], function (err) {
                if (err) {
                    self.log.info(err);
                } else {
                    self.log.info('server ' + newServer.id + ' added successfully');
                }
                callback(err);
            });
        }
    };

    this.updateServerInternal = function (serverConfig, callback) {
        var cfg = cfgToDBCfg(serverConfig);

        async.waterfall([
            function (cb) {
                // 1. update db
                dibs.rpc.datamgr.updateServer(cfg, function (err) {
                    cb(err);
                });
            },
            function (cb) {
                getConfigFromDB({}, cb);
            },
            function (configs, cb) {
                // 2. apply db
                applyConfigFromDB(configs, cb);
            }
        ], function (err) {
            callback(err);
        });
    };

    this.removeServerInternal = function (serverId, options, callback) {
        self.log.info('remove server ' + serverId);
        self.log.info(options);
        var server = _.clone(dibs.getServer(serverId));
        if (server.status === 'RUNNING' || server.status === 'DISCONNECTED') {
            async.waterfall([
                function (cb) {
                    if (server.status === 'RUNNING') {
                        self.log.info(' - stop server ' + serverId);
                        self.stopServerInternal(serverId, cb);
                    } else {
                        cb(null);
                    }
                },
                function (cb) {
                    if (options.cleanResource) {
                        if (utils.isLocalAddr(server.host)) {
                            self.log.info(' - clean local server resource ' + serverId);
                            self.removeServerResource(server, cb);
                        } else {
                            self.log.info(' - clean remote server resource ' + serverId);
                            self.removeRemoteServerResource(server, cb);
                        }
                    } else {
                        cb(null);
                    }
                },
                function (cb) {
                    // 1. update db
                    var server = dibs.getServer(serverId);
                    dibs.removeServer(server);
                    self.log.info(' - remove config from DB');
                    dibs.rpc.datamgr.deleteServer(serverId, function (err) {
                        cb(err);
                    });
                }
            ], function (err) {
                if (err) {
                    self.log.info(err);
                } else {
                    self.log.info('server ' + serverId + ' removed successfully');
                }
                callback(err);
            });
        } else {
            callback(new Error('server ' + serverId + ' is not stopable status (RUNNING/DISCONNECTED)\n try again a few minutes later'));
        }
    };

    this.createRemoteServerConfig = function (server, config, callback) {
        if (server.agentServer) {
            var handler = {
                try: function () {
                    server.agentServer.connectAgentServer(server, self, function (err) {
                        if (err) {
                            callback(err);
                        } else {
                            server.agentServer.createServerConfig(server, config, callback);
                        }
                    });
                },
                catch: function (err) {
                    self.log.error('create remote Server Config using Agent Failed : ' + server.id);
                    self.log.error(err);
                    callback(err);
                }
            };
            DHandle.run(handler);
        } else {
            self.log.error('agentServer instance is null ... ' + server.id);
            callback(new DError('MASTER004', {
                ip: server.host
            }));
        }
    };

    this.removeServerResource = function (server, callback) {
        if (server.config && server.config.getConfigData) {
            var config = server.config.getConfigData();
            var resourcePath = path.join(config.config_root_dir, config.id);
            utils.removePathIfExist(resourcePath, callback);
        } else {
            callback(new Error('server ' + server.name + 'config information is not exist'));
        }
    };

    this.removeRemoteServerResource = function (server, callback) {
        if (server.agentServer) {
            var handler = {
                try: function () {
                    server.agentServer.connectAgentServer(server, self, function (err) {
                        if (err) {
                            callback(err);
                        } else {
                            server.agentServer.removeServerResource(server, callback);
                        }
                    });
                },
                catch: function (err) {
                    self.log.error('remove remote Server resource using Agent Failed : ' + server.id);
                    self.log.error(err);
                    callback(err);
                }
            };
            DHandle.run(handler);
        } else {
            self.log.error('agentServer instance is null ... ' + server.id);
            callback(new DError('MASTER004', {
                ip: server.host
            }));
        }
    };


    //
    // PRIVATE
    //

    function cfgToDBCfg(config) {
        var cfg = {};
        var cConfig = config;
        cConfig = _.omit(cConfig, 'sub_servers');
        cfg.id = cConfig.id;
        cConfig = _.omit(cConfig, 'id');
        cfg.type = cConfig.type;
        cConfig = _.omit(cConfig, 'type');
        cfg.os_type = cConfig.os_type;
        cConfig = _.omit(cConfig, 'os_type');
        cfg.parent_id = cConfig.parent_id;
        cConfig = _.omit(cConfig, 'parent_id');
        cfg.host = cConfig.host;
        cConfig = _.omit(cConfig, 'host');
        cfg.port = cConfig.port;
        cConfig = _.omit(cConfig, 'port');
        cfg.domain_name = cConfig.domain_name;
        cConfig = _.omit(cConfig, 'domain_name');
        cfg.description = cConfig.description;
        cConfig = _.omit(cConfig, 'description');
        cfg.information = cConfig;
        return cfg;
    }

    function setConfigToDB(config, callback) {
        var cfg = cfgToDBCfg(config);

        dibs.rpc.datamgr.addServer(cfg.id, cfg.type, cfg.parent_id, cfg.os_type, cfg.host, cfg.port, cfg.domain_name, cfg.description, cfg.information, callback);
    }

    function getConfigFromDB(condition, callback) {
        dibs.rpc.datamgr.searchServers(condition, function (err, configs) {
            if (err) {
                callback(err, null);
            } else {
                var newConfigs = _.map(configs, function (config) {
                    _.each(config.information, function (value, key) {

                        config[key] = value;
                    });
                    return _.omit(config, 'information');
                });
                callback(err, newConfigs);
            }
        });
    }


    function applyConfigFromDB(dbConfigs, callback) {
        var masterConfig = getConfigFromServer(self);
        var mConfig = _.clone(masterConfig);
        mConfig = _.omit(mConfig, 'sub_servers');
        var subServers = masterConfig['sub_servers'];
        var memServerConfigs = _.flatten(_.values(subServers), true);
        memServerConfigs.push(mConfig);

        var dbServerConfigs = _.filter(dbConfigs, function (config) {
            var serverTypes = _.map(dibs.plugin.getExtensions('dibs.base.server'), function (server) {
                return server.attributes.serverType;
            });
            return (_.indexOf(serverTypes, config.type) >= 0);
        });


        // 3. compare server configs from db
        var memKeys = _.map(memServerConfigs, genConfigKey);

        var dbKeys = _.map(dbServerConfigs, genConfigKey);

        var updateKeys = _.intersection(memKeys, dbKeys);
        var addKeys = _.difference(dbKeys, updateKeys);
        var removeKeys = _.difference(memKeys, updateKeys);

        // update memory using db information
        _.each(updateKeys, function (key) {
            var info = parseConfigKey(key);
            var server = dibs.getServer(info.id);
            var newConf = _.filter(dbConfigs, function (conf) {
                return (conf.id === info.id);
            })[0];
            if (newConf.type === 'master') {
                newConf['sub_servers'] = _.filter(dbServerConfigs, function (config) {return config.type !== 'master';});
            }
            server.config = dibs.config.newServerConfig(server, newConf);
        });

        // remove servers
        var removeServers = _.compact(_.map(removeKeys, getServerFromConfigKey));

        var addServerConfs = _.filter(dbConfigs, function (cfg) {
            return (_.indexOf(addKeys, genConfigKey(cfg)) >= 0);
        });

        async.waterfall([
            function (cb) {
                async.eachSeries(removeServers, stopSubServer, function (err) {
                    cb(err);
                });
            },
            function (cb) {
                async.eachSeries(addServerConfs, applyServer, function (err) {
                    cb(err);
                });
            }
        ], function (err) {
            callback(err);
        });

    }

    function genConfigKey(config) {
        return config.id + ':' + config.host + ':' + config.port + ':' + config.type;
    }

    function parseConfigKey(key) {
        var info = key.split(':');
        return {
            id: info[0],
            host: info[1],
            port: info[2],
            type: info[3]
        };
    }

    function getServerFromConfigKey(key) {
        return dibs.getServer(parseConfigKey(key).id);
    }

    function getConfigFromServer(server) {
        if (server && server.config) {
            if (server.config.getConfigData) {
                return server.config.getConfigData();
            } else {
                return server.config;
            }
        } else {
            return null;
        }
    }


    function setConfigFromRealServers(servers, callback) {
        async.each(servers, setConfigFromRealServer, callback);
    }

    function setConfigFromRealServer(server, callback) {
        server.__getServerConfiguration(function (err, config) {
            if (err) {
                self.log.warn('Fail get server configurations: ' + server.id + ' - ' + err);
            } else {
                var masterConfigs = getConfigFromServer(self);
                var subServer = (masterConfigs) ? masterConfigs['sub_servers'] : null;
                var oldServer = (subServer) ? subServer[server.type] : null;
                if (_.isArray(oldServer)) {
                    oldServer = _.filter(oldServer, function (serv) {
                        return serv.id === server.id;
                    })[0];
                }

                if (config) {
                    if (oldServer && oldServer.host) {
                        config.host = oldServer.host;
                    }
                    if (oldServer && oldServer.agent) {
                        config.agent = oldServer.agent;
                    }
                    if (oldServer && oldServer['web_port']) {
                        config['web_port'] = oldServer['web_port'];
                    }
                }

                server.config = config;
            }
            callback(err);
        });
    }

    function getConfigFromServers(servers) {
        return _.compact(_.map(servers, getConfigFromServer));
    }


    function applyServer(config, callback) {
        console.log(config.id + ' will start');
        addServer(config.type, config);
        var server = dibs.getServer(config.id);
        server.config = dibs.config.newServerConfig(server, config);
        server.id = config.id;

        async.waterfall([
            function (cb) {
                if (utils.isLocalAddr(server.host)) {
                    self.createServerConfig(server, config);
                    cb(null);
                } else {
                    self.createRemoteServerConfig(server, config, cb);
                }
            },
            function (cb) {
                if (utils.isLocalAddr(server.host)) {
                    startLocalServers([server], function (err) {
                        cb(err);
                    });
                } else {
                    startRemoteServers([server], function (err) {
                        cb(err);
                    });
                }
            },
            function (cb) {
                monitorSubServer(server.id);
                activeServers[server.id] = server;
                monitorServerResourceUsage(server.id);
                cb(null);
            }
        ], function (err) {
            if (err) {
                console.log(err);
            }
            callback(err);
        });
    }

    function initiateToMonitorServers(servers) {

        // Monitor server periodically
        async.each(servers,
            function (s, cb) {
                self.log.info('Starting to monitor server... ' + s.id);
                monitorSubServer(s.id);

                monitorServerResourceUsage(s.id);
                cb(null);
            },
            function (err) {
                if (err) {
                    self.log.error(err);
                }
                self.log.info('Monitoring all sub servers has been in-prgress.');
            });

        // If server status are changed, broad cast their status
        broadcastServerStatusPeriodically(1000);
    }


    // by ms
    function broadcastServerStatusPeriodically(period) {
        var old = [];
        var oldConfs = [];

        self.addScheduledAction('broadcast-server-status', {
            period: period
        }, function (cb) {
            if (self.status !== 'RUNNING' && self.status !== 'INITIALIZED') {
                cb(null); return;
            }

            var servers = dibs.getAllServers();
            // check status changed
            var latest = _.pluck(servers, 'status');
            var latestConfs = _.pluck(servers, 'config');

            if (old.length !== latest.length ||
                !old.every(function (u, i) {
                    return u === latest[i];
                })) {

                old = latest;
                broadCastSubServerStatus(cb);
            } else if (oldConfs.length !== latestConfs.length ||
                !oldConfs.every(function (u, i) {
                    return u === JSON.stringify(latestConfs[i]);
                })) {

                oldConfs = _.map(latestConfs, function (c) {
                    return JSON.stringify(c);
                });
                broadCastSubServerStatus(cb);
            } else {
                cb(null);
            }
        });
    }


    function migrateConfigs(callback) {
        var servers = dibs.getAllServers();
        var configs = getConfigFromServers(servers);

        async.eachSeries(configs, function (config, cb) {
            self.log.info('Add ' + config.id + ' configs to DB');
            if (!config.parent_id) {
                config.parent_id = config.host;
            }
            setConfigToDB(config, cb);
        }, function (err) {
            callback(err);
        });
    }

    function getCoreServers() {
        return _.filter(dibs.getAllServers(), function (server) {
            return (server.name !== 'builder' && server.name !== 'test');
        });
    }

    function waitForAllServersAreReady(callback) {
        var servers = dibs.getAllServers();

        async.series([
            function (cb) {
                self.log.info('Waiting for RUNNING states...');
                waitForServersAreRUNNING(servers, cb);
            }],
            function (err) {
                callback(err);
            });
    }


    function waitForCoreServersAreReady(callback) {
        var servers = getCoreServers();

        async.series([
            function (cb) {
                self.log.info('Waiting for RUNNING states...');
                waitForServersAreRUNNING(servers, cb);
            }],
            function (err) {
                callback(err);
            });
    }


    function waitForServersAreRUNNING(servers, callback) {
        var retryCnt = 0;

        // Check server init
        var exec = setInterval(function () {
            retryCnt++;
            async.every(servers, function (srv, ecb) {
                if (srv.id === self.id) {
                    ecb(true);
                } else {
                    ecb(srv.status === 'RUNNING');
                }
            }, function (result) {
                if (result) {
                    clearInterval(exec);
                    callback(null);
                } else {
                    // wait for 10 minutes
                    if (retryCnt > 1200) {
                        clearInterval(exec);
                        callback(new DError('MASTER006'));
                    }
                }
            });
        }, 1000);
    }


    function waitForServerIsReady(sid, callback) {
        var retryCnt = 0;

        // Check server init
        var exec = setInterval(function () {
            var server = dibs.getServer(sid);

            if (server) {
                retryCnt++;
                if (server.status === 'RUNNING') {
                    clearInterval(exec);
                    callback(null);
                } else {
                    // wait for 5 minutes
                    if (retryCnt > 300) {
                        clearInterval(exec);
                        callback(new DError('MASTER007'));
                    }
                }
            } else {
                clearInterval(exec);
                callback(new DError('MASTER017', { serverId: sid }));
            }
        }, 1000);
    }


    function waitForServerIsDisconnected(sid, callback) {
        var server = dibs.getServer(sid);
        var retryCnt = 0;

        // Check server init
        var exec = setInterval(function () {
            retryCnt++;
            if (server.status === 'DISCONNECTED') {
                clearInterval(exec);
                callback(null);
            } else {
                // wait for 5 minutes
                if (retryCnt > 300) {
                    clearInterval(exec);
                    callback(new DError('MASTER008'));
                }
            }
        }, 1000);
    }

    function notifyServerResourceUsage(serverId, server, callback) {
        var subject = '[' + serverId + '] Server Resource Usage';
        var msg = 'Disk space of \'' + serverId + '\' has reached ' + server.diskUsage + '%.' +
            ' Please secure enough disk space.\n\n' +
            '<< Information >>' + '\n' +
            '- Server ID: ' + serverId + '\n' +
            '- CPU Usage: ' + server.cpuUsage + '%\n' +
            '- Memory Usage: ' + server.memUsage + '%\n' +
            '- Disk Usage: ' + server.diskUsage + '%\n';

        async.waterfall([
            function (cb) {
                dibs.rpc.datamgr.searchUsersByGroupName('administrator', function (err, users) {
                    var targets = null;
                    if (err) {
                        self.log.error('failed to search users by group name');
                    } else {
                        targets = _.without(_.pluck(users, 'email'), 'admin@user');
                    }
                    cb(err, targets);
                });
            },
            function (targets, cb) {
                dibs.rpc.messenger.notify('email', subject, targets, msg, function (err) {
                    if (err) {
                        self.log.error('failed to send the notification email for server resources.');
                    }
                    cb(err);
                });
            }
        ],
            function (err) {
                if (err) {
                    self.log.error(err);
                }
                callback(err);
            });
    }

    this.checkServerResourceUsage = function (server) {
        if (server === null) {
            return false;
        }

        return (server.diskUsage >= DISKSPACE_LIMIT);
    };

    this.registerServerInfoInternal = function (serverId, info, callback) {
        var server = dibs.getServer(serverId);
        if (server) {
            dibs.log.info('Register server(' + serverId + ') informations');
            server.environments = info.environments;
            callback(null);
        } else {
            callback(new DError('MASTER017', {
                serverId: serverId
            }));
        }
    };

    function monitorServerResourceUsage(serverId) {

        self.addScheduledAction('notify-resource-' + serverId, {
            period: NOTIFICATION_DELAY
        }, function (cb) {
            var server = dibs.getServer(serverId);

            if (server) {
                if (server.diskspaceShortage && dibs.getServersByType('messenger')[0]) {
                    self.log.info('notify server resource usage of \'' + serverId + '\' to admin.');
                    notifyServerResourceUsage(serverId, server, function (err) {
                        cb(null);
                    });
                } else {
                    self.log.info('ignore server resource usage of \'' + serverId);
                    cb(null);
                }
            } else {
                self.log.warn('failed to get server using ' + serverId);
                cb(null);
            }
        });
    }

    function monitorSubServer(sid) {
        self.addScheduledAction('monitor-server-' + sid, {
            startTime: new Date((new Date()).getTime() + 1000),
            period: 1000 // 1 sec
        }, function (cb) {
            var server = dibs.getServer(sid);

            if (!server) {
                self.removeScheduledAction('monitor-server-' + sid);
                self.removeScheduledAction('notify-resource-' + sid);
                return cb(null);
            }

            // skip checking if not active
            if (!activeServers[server.id]) {
                return cb(null);
            }

            getServerStatus(server, function (err, result) {
                // When disconnected, write the log of status-change
                if (err && result.status === 'DISCONNECTED' &&
                    server.status !== 'DISCONNECTED') {

                    self.log.warn('Getting the status of server \'' + server.id + '\' failed!');
                    self.log.warn(err);
                }

                // accumulate disconnected time
                if (result.status === 'DISCONNECTED') {
                    if (disconnectedTimes[server.id]) {
                        disconnectedTimes[server.id]++;
                    } else {
                        disconnectedTimes[server.id] = 1;
                    }
                } else {
                    disconnectedTimes[server.id] = 0;
                }

                // update server status
                server.status = result.status;
                server.cpuUsage = result.cpuUsage;
                server.memUsage = result.memUsage;
                server.diskUsage = result.diskUsage;

                server.diskspaceShortage = self.checkServerResourceUsage(server);

                // if disconnected time is more than 3 min, recover the server
                if (disconnectedTimes[server.id] && disconnectedTimes[server.id] > 60 * 3) {
                    self.log.warn('Server \'' + server.id + '\' is disconnected by internal error!');
                    self.log.warn('Recovering server \'' + server.id + '\'...');
                    // NOTE. MUST reset disconnected time. if not, try to launch server at every sec
                    disconnectedTimes[server.id] = 0;
                    async.series([
                        function (cb1) {
                            self.log.info('Unregistering DFS server if required... ' + server.id);
                            dfs.masterIndex.unregisterServer(server.id, cb1);
                        },
                        function (cb1) {
                            self.log.info('Launching sub-server... ' + server.id);
                            launchServer(server, cb1);
                        }
                    ], function (err) {
                        notify(server.id, err);
                        cb(err);
                    });
                } else {
                    cb(null);
                }
            });
        });
    }

    function notify(serverId, result) {
        self.log.info('wait for server reboot');
        waitForServerIsReady(serverId, function (err) {
            if (err) {
                self.log.info('failed waitting server ready');
                self.log.info(err);
                return;
            }
            var subject = '[' + self.id + '] Recovering server';
            if (!result && dibs.rpc.messenger && dibs.rpc.datamgr) {
                sendEmail(subject + ' succeeded', serverId, null, function () {});
            } else if (result && dibs.rpc.messenger && dibs.rpc.datamgr) {
                sendEmail(subject + ' failed', serverId, result, function () {});
            } else {
                self.log.info('ignore sending email');
            }
        });
    }

    function sendEmail(subject, serverId, launchResult, callback) {
        var msg = '';
        async.waterfall([
            function (cb) {
                self.log.info('connect to mongo db');
                var logDbHost = dibs.thisServer.config.get('remote_log_db_host') || null;
                var logDbName = dibs.thisServer.config.get('remote_log_db_name') || null;
                var debug = dibs.thisServer.config.get('debug') || false;

                var logger = dibs.log.openMonitor(serverId, {
                    remoteLogDbHost: logDbHost,
                    remoteLogDbName: logDbName,
                    debug: debug
                });

                logger.getLastNLine(serverId, {}, 400, cb);
            },
            function (docs, cb) {
                if (launchResult) {
                    msg += 'Server recover failed\n' + 'Target: ' + serverId + '\n\n' +
                        'Launch Error Message: \n' + launchResult.message + '\n\n' +
                        'Slerver Log: \n' +
                        ' ... \n';
                } else {
                    msg += 'Server recover succeeded\n' + 'Target: ' + serverId + '\n\n' +
                        'Server Log: \n' +
                        ' ... \n';
                }
                _.each(_.sortBy(docs, function (d) {
                    return d.timestamp;
                }), function (doc) {
                    msg += doc.timestamp + ' ' + doc.level + ' ' + doc.message + '\n';
                });

                self.log.info('get admin list');
                dibs.rpc.datamgr.searchUsersByGroupName('administrator', cb);
            },
            function (rst, cb) {
                self.log.info('send email to admin');
                var targets = _.pluck(rst, 'email');
                if (dibs.getServersByType('messenger')[0]) {
                    dibs.rpc.messenger.notify('email', subject, _.without(targets, 'admin@user'), msg, cb);
                } else {
                    cb(new Error('messenger server is not exists'));
                }
            }], function (err) {
            if (err) {
                self.log.info('ignore sending email');
                self.log.info(err);
            }
            callback(err);
        });
    }

    function getServerStatus(s, cb) {
        if (s.id !== self.id) {
            s.__getStatus(function (err, status) {
                if (err) {
                    cb(err, {
                        status: 'DISCONNECTED',
                        cpuUsage: 0,
                        memUsage: 0,
                        diskUsage: 0
                    });
                } else {
                    cb(err, status);
                }
            });
        } else {
            self.getStatus(function (err, status) {
                if (err) {
                    self.log.error('master server: getStatus() failure');
                }
                cb(err, status);
            });
        }
    }


    function broadCastSubServerStatus(callback) {
        var servers = dibs.getAllServers();
        var msg = servers.map(function (e) {
            var conf = getConfigFromServer(e);
            return {
                id: e.id,
                host: e.host,
                port: e.port,
                type: e.type,
                status: e.status,
                platform: e.platform,
                arch: e.arch,
                totalmem: e.totalmem,
                freemem: e.freemem,
                environments: e.environments,
                config: conf
            };
        });

        self.log.info('Broadcast server informations.');
        async.each(servers,
            function (server, cb) {
                if (!server.isOnline()) {
                    cb(null);return;
                }
                if (server.id === self.id) {
                    var subServers = _.filter(servers, function (s) {
                        return s.id !== self.id;
                    });

                    var masterConfigs = getConfigFromServer(self);
                    var oldSubServers = (masterConfigs) ? masterConfigs['sub_servers'] : null;

                    var subConfigs = {};
                    _.each(subServers, function (s) {
                        if (!subConfigs[s.type]) {
                            subConfigs[s.type] = [];
                        }

                        var oldServer = (oldSubServers) ? oldSubServers[s.type] : null;
                        if (_.isArray(oldServer)) {
                            oldServer = _.filter(oldServer, function (serv) {
                                return (serv && serv.id === s.id);
                            })[0];
                        }
                        var newConf = getConfigFromServer(s);

                        if (newConf) {
                            if (oldServer && oldServer.host) {
                                newConf.host = oldServer.host;
                            }
                            if (oldServer && oldServer.agent) {
                                newConf.agent = oldServer.agent;
                            }
                            if (oldServer && oldServer['web_port']) {
                                newConf['web_port'] = oldServer['web_port'];
                            }
                            subConfigs[s.type].push(newConf);
                        } else {
                            if (self.status !== 'RUNNING') {
                                subConfigs[s.type].push(oldServer);
                            }
                        }
                    });
                    var oldConfig = JSON.stringify(getConfigFromServer(server));
                    if (self.config.set) {
                        self.config.set('sub_servers', subConfigs);
                    } else {
                        self.config['sub_servers'] = subConfigs;
                    }
                    if (oldConfig !== JSON.stringify(getConfigFromServer(self))) {
                        if (server.config.set) {
                            server.config.set('sub_servers', subConfigs);
                        } else {
                            server.config['sub_servers'] = subConfigs;
                        }
                        self.createServerConfig(self, getConfigFromServer(self));
                        cb(null);return;
                    } else {
                        cb(null);return;
                    }
                }

                server.__updateServerInformation(msg, function (err) {
                    if (err) {
                        self.log.warn('Failed broadcast server staus to \'' + server.id + '\'!');
                        self.log.warn(err);
                    }
                    cb(err);
                });
            },
            function (err) {
                callback(err);
            });
    }


    // start child-servers
    function launchServer(server, callback) {
        if (utils.isLocalAddr(server.host)) {
            launchLocalServer(server, callback);
        } else {
            if (server.agentServer) {
                var handler = {
                    try: function () {
                        server.agentServer.connectAgentServer(server, self, function (err) {
                            if (err) {
                                callback(err);
                            } else {
                                self.log.info('Starting remote server execution ... ' + server.id);
                                executeRemoteServer(server, function (fake, err) {
                                    callback(err);
                                });
                            }
                        });
                    },
                    catch: function (err) {
                        self.log.error('execute remote Single Server using Agent Failed : ' + server.id);
                        self.log.error(err);
                        callback(err);
                    }
                };
                DHandle.run(handler);
            } else {
                self.log.error('agentServer instance is null ... ' + server.id);
                callback(new DError('MASTER004', { ip: server.host }));
            }
        }
    }


    function relaunchServer(server, message, callback) {
        self.log.info('Relaunching sub-server... ' + server.id);
        relaunchSubServer(server, function (err) {
            if (err) {
                if (err.inner) {
                    console.log(message + err.inner.message);
                } else {
                    console.log(message + err.message);
                }
            } else {
                server.status = 'INITIALIZED';
                console.log(message + 'relaunch sub server success');
            }

            callback(null, err);
        });
    }


    function startLocalServersIgnoreBuilderError(localServers, callback) {
        startLocalServers(localServers, function (err, errors) {
            var newError = [];
            _.each(errors, function (error) {
                if (error) {
                    console.log(error);
                }
                newError.push(null);
            });
            callback(err, newError);
        });
    }

    function startLocalServers(localServers, callback) {
        async.map(localServers, function (server, ecb) {
            self.log.info('Starting local server execution ... ' + server.id);
            executeLocalServer(server, ecb);
        }, callback);
    }


    function executeLocalServer(server, callback) {
        launchLocalServer(server, function (err) {
            var message = '  - ' + server.id + ' ... ';

            if (err) {
                if (err.inner) {
                    console.log(message + err.inner.message);

                    // 'address is already in used'
                    if (err.inner.message.indexOf('ERR002') !== -1 && self.startOptions.force) {
                        return relaunchServer(server, message, callback);
                    }
                } else {
                    console.log(message + err.message);
                }
            } else {
                server.status = 'INITIALIZED';
                console.log(message + 'local server success');
            }
            callback(null, err);
        });
    }


    function startRemoteServersIgnoreBuilderError(remoteServers, callback) {
        startRemoteServers(remoteServers, function (err, errors) {
            var newError = [];
            _.each(_.flatten(errors, true), function (error) {
                if (error) {
                    console.log(error);
                }
                newError.push(null);
            });
            callback(err, newError);
        });
    }

    function startRemoteServers(remoteServers, callback) {
        if (_.isEmpty(remoteServers)) {
            return callback(null, []);
        }

        var serverHosts = [];
        _.each(remoteServers, function (server) {
            if (server.agentServer) {
                var domain = server.agentServer.host + ':' + server.agentServer.port;
                serverHosts.push({
                    id: domain
                });
            }
        });
        serverHosts = _.uniq(_.pluck(serverHosts, 'id'));

        var agentServers = [];
        _.each(serverHosts, function (serverHost) {
            var agentHost = serverHost.split(':')[0];
            var agentPort = Number(serverHost.split(':')[1]);

            var group = [];
            _.each(remoteServers, function (server) {
                if ((server.agentServer.host === agentHost) && (server.agentServer.port === agentPort)) {
                    group.push(server);
                }
            });

            agentServers.push({
                id: serverHost,
                group: group
            });
        });

        async.map(agentServers, function (server, cb) {
            // update DIBS module via agent server
            self.log.info('Starting agent server execution ... ' + server.group[0].id);
            var agentServer = server.group[0].agentServer;

            if (agentServer) {
                var handler = {
                    try: function () {
                        agentServer.connectAgentServer(server.group[0], self, function (err) {
                            if (err) {
                                cb(null, err);
                            } else {
                                async.map(server.group, function (svr, ecb) {
                                    self.log.info('Starting remote server execution ... ' + svr.id);
                                    executeRemoteServer(svr, ecb);
                                }, cb);
                            }
                        });
                    },
                    catch: function (err) {
                        self.log.error('execute remote Server using Agent Failed : ' + server.group[0].id);
                        self.log.error(err);
                        cb(null, err);
                    }
                };
                DHandle.run(handler);
            } else {
                self.log.error('agentServer instance is null ... ' + server.group[0].id);
                cb(null, new DError('MASTER004', { ip: server.group[0].host }));
            }
        }, callback);
    }


    function executeRemoteServer(server, callback) {
        var startOptions = {
            masterAddr: dibs.thisServer.host + ':' + dibs.thisServer.port,
            cleanStart: self.startOptions.cleanStart,
            debug: self.startOptions.debug
        };

        if (server.agentServer) {
            server.agentServer.launchServer(server, self, startOptions, function (err) {
                var message = '  - ' + server.id + ' ... ';

                if (err) {
                    if (err.inner) {
                        console.log(message + err.inner.message);

                        // 'address is already in used'
                        if (err.inner.message.indexOf('ERR002') !== -1 && self.startOptions.force) {
                            return relaunchServer(server, message, callback);
                        }
                    } else {
                        console.log(message + err.message);
                    }
                } else {
                    server.status = 'INITIALIZED';
                    console.log(message + 'remote server success');
                }
                callback(null, err);
            });
        } else {
            self.log.error('agentServer instance is null ... ' + server.id);
            callback(null, new DError('MASTER004', {
                ip: server.host
            }));
        }
    }


    function launchLocalServer(server, callback) {
        var availableCallback = true;
        // get app.js's path
        var appPath = path.join(__dirname, '..', '..', 'app.js');
        var child;
        var innerError = null;
        var masterAddr = dibs.thisServer.host + ':' + dibs.thisServer.port;

        var nodeOptions = [];
        if (self.startOptions.debug) {
            var debugPort = Number(server.port) + 1000;
            nodeOptions.push('--stack-size=1024');
            nodeOptions.push('--debug=' + debugPort);
            console.log('  -> ' + server.id + ' debug port: ' + debugPort);
        }
        else {
            nodeOptions = ['--stack-size=1024'];
        }

        if (self.startOptions.maxOldSpaceSize) {
            // use v8 option --max_old_space_size to avoid 'CALL_AND_RETRY_LAST Allocation failed' error.
            nodeOptions.push('--max_old_space_size=' + self.startOptions.maxOldSpaceSize);
        }

        if (self.startOptions.cleanStart) {
            // DEBUG CODE FOR MEMORY LEAK
            //child = spawn("node", ["--expose-gc", appPath, '-i', server.id, '-t', server.type, '-p', server.port, '--cleanStart', '--masterAddr', masterAddr]);
            nodeOptions = nodeOptions.concat([appPath, '-i', server.id, '-t', server.type, '-p', server.port, '--cleanStart', '--masterAddr', masterAddr]);
        } else {
            // DEBUG CODE FOR MEMORY LEAK
            //child = spawn("node", ["--expose-gc", appPath, '-i', server.id, '-t', server.type, '-p', server.port, '--masterAddr', masterAddr]);
            nodeOptions = nodeOptions.concat([appPath, '-i', server.id, '-t', server.type, '-p', server.port, '--masterAddr', masterAddr]);
        }

        child = spawn('node', nodeOptions);
        child.stdout.on('data', function (out) {
            var id = server.id;
            if (out.toString().indexOf('Server is LISTENING!') !== -1) {
                if (availableCallback) {
                    availableCallback = false;
                    callback(null);
                }
            } else {
                self.log.info('[' + id + '] ' + out.toString());
            }
        });

        child.stderr.on('data', function (err) {
            var id = server.id;
            if (/^execvp\(\)/.test(err)) {
                console.error('Failed to start child process: ' + id);
            }
            innerError = new Error(err.toString().split(os.EOL).join(''));
            self.log.error('[' + id + '] ' + err);
        });

        child.on('close', function (code) {
            if (code !== 0) {
                if (availableCallback) {
                    availableCallback = false;
                    if (innerError) {
                        callback(new DError('MASTER009', {
                            id: server.id,
                            code: code
                        }, innerError));
                    } else {
                        callback(new DError('MASTER009', {
                            id: server.id,
                            code: code
                        }));
                    }
                }
            }
        });
    }


    function addServers(servers) {
        dibs.addServer(self);
        for (var serverType in servers) {
            // check server type exist
            if (!dibs.existsServerType(serverType)) {
                throw new DError('MASTER010', {
                    type: serverType
                });
            }
            var serverInfo = null;
            if (Object.prototype.toString.call(servers[serverType]) === '[object Array]') {
                for (var i = 0; i < servers[serverType].length; i++) {
                    serverInfo = servers[serverType][i];
                    if (serverInfo) {
                        addServer(serverType, serverInfo);
                    }
                }
            } else {
                serverInfo = servers[serverType];
                if (serverInfo) {
                    addServer(serverType, serverInfo);
                }
            }
        }
    }

    function addServer(type, serverInfo) {
        var tmpServer = dibs.createServer(serverInfo.id, type);
        tmpServer.host = serverInfo.host;
        tmpServer.port = serverInfo.port;
        if (!tmpServer.environments) tmpServer.environments = [];

        if (useAgentServer(serverInfo)) {
            // generate agent server information
            var tempAgent;
            if (serverInfo.agent && isNaN(Number(serverInfo.agent))) {
                dibs.log.error('[' + serverInfo.id + '] Agent port type is not correct:' + serverInfo.agent);
                dibs.log.error('[' + serverInfo.id + '] Skip addServer');
            } else if (serverInfo.agent && !isNaN(Number(serverInfo.agent))) {
                // use specified port information
                tempAgent = dibs.createServer(serverInfo.host + '_agent_' + serverInfo.agent, 'agent');
                tempAgent.host = serverInfo.host;
                tempAgent.port = Number(serverInfo.agent);
                tmpServer.agentServer = tempAgent;
                dibs.addServer(tmpServer);
            } else {
                tempAgent = dibs.createServer(serverInfo.host + '_agent_' + DEFAULT_AGNET_PORT, 'agent');
                tempAgent.host = serverInfo.host;
                tempAgent.port = DEFAULT_AGNET_PORT;
                tmpServer.agentServer = tempAgent;
                dibs.addServer(tmpServer);
            }
        } else {
            dibs.addServer(tmpServer);
        }
    }


    function useAgentServer(serverInfo) {
        if (utils.isLocalAddr(serverInfo.host)) {
            return !(!serverInfo.agent);
        } else {
            return true;
        }
    }


    function relaunchSubServer(server, callback) {
        async.series([
            function (cb) {
                self.log.info('Unregistering DFS server if required... ' + server.id);
                dfs.masterIndex.unregisterServer(server.id, cb);
            },
            function (cb) {
                self.log.info('Restarting sub-server... ' + server.id);
                server.terminateServer(cb);
            },
            function (cb) {
                setTimeout(function () {
                    cb(null);
                }, 3000);
            },
            function (cb) {
                self.log.info('Launching sub-server... ' + server.id);
                launchServer(server, cb);
            }
        ], function (err) {
            callback(err);
        });
    }


    function isLaunchedUsingSourceCode() {
        if (!fs.existsSync(path.join(__dirname, '..', '..', '.info'))) {
            return true;
        } else {
            return false;
        }
    }


    function installAndforkNewServer(installDir, callback) {
        var updatesDir = path.join(__dirname, '..', '..', 'updates');
        async.series([
            // check install/update directory
            // create DIBS packages
            function (cb) {
                if (!fs.existsSync(installDir)) {
                    extfs.mkdirsSync(installDir);
                }
                if (fs.existsSync(updatesDir)) {
                    extfs.removeSync(updatesDir);
                }
                extfs.mkdirsSync(updatesDir);

                dibs.log.info('Creating DIBS package files from source code...');
                createDIBSPackages(updatesDir, cb);
            },
            // install DIBS packages to install directory
            function (cb) {
                dibs.log.info('Installing DIBS package files to install directory...');
                installDIBSPackages(updatesDir, installDir, cb);
            },
            // launch server on install directory
            function (cb) {
                dibs.log.info('Launching new installed master server...');
                forkNewMasterServer(installDir, {}, cb);
            }
        ], function (err) {
            if (err) {
                callback(err);
            } else {
                dibs.log.info('Terminating server after forking installed master server...');
                callback(new DError('MASTER014'));
            }
        });
    }


    function createDIBSPackages(targetDir, callback) {
        var dibsDir = path.join(__dirname, '..', '..');
        var monitor = new Monitor({
            onProgress: function (info, cb) {
                dibs.log.info(info.log);
            }
        });
        Builder.build({}, dibsDir, monitor, function (err) {
            if (!err) {
                var pkgFile = fs.readdirSync(dibsDir).filter(function (e) {
                    return e.indexOf('dibs') !== -1;
                })[0];
                FileSystem.move(path.join(dibsDir, pkgFile),
                    path.join(targetDir, pkgFile), callback);
            } else {
                callback(err);
            }
        });
    }


    function installDIBSPackages(updatesDir, installDir, callback) {
        var monitor = new Monitor({
            onProgress: function (info, cb) {
                dibs.log.info(info.log);
            }
        });
        var pkgFiles = fs.readdirSync(updatesDir).map(function (e) {
            return path.join(updatesDir, e);
        });
        Installer.installPackages(pkgFiles, installDir, {
            force: true
        }, monitor, callback);
    }


    function forkNewMasterServer(installDir, options, callback) {
        forkNewMasterServerInternal(installDir, options, function (err) {
            if (err) {
                // retry once again after 2 sec
                dibs.log.info('Retrying to launch new master server... ');
                setTimeout(function () {
                    forkNewMasterServerInternal(installDir, options, callback);
                }, 2000);
            } else {
                callback(err);
            }
        });
    }


    function forkNewMasterServerInternal(installDir, options, callback) {
        var args = ['--stack-size=1024', 'app.js', '-i', self.id, '-p', self.port];
        if (self.startOptions.cleanStart) {
            args.push('--cleanStart');
        }
        if (options.update) {
            args.push('--update');
            args.push(options.update);
        }
        var child = spawn('node', args, {
            detached: true,
            cwd: installDir,
            stdio: 'inherit'
        });
        // NOTE. Need some delay
        var t = setTimeout(function () {
            child.unref();
            callback(null);
        }, 1000);

        child.on('exit', function (code, signal) {
            clearTimeout(t);
            callback(new Error('Creating process failed!'));
        });

    }


    function existsUpdates() {
        if (fs.existsSync(path.join(__dirname, '..', '..', 'updates')) &&
            fs.readdirSync(path.join(__dirname, '..', '..', 'updates')).length > 0) {
            return true;
        } else {
            return false;
        }
    }


    function isLaunchedUsingUpdateMode() {
        if (self.startOptions.update) {
            return true;
        } else {
            return false;
        }
    }


    function forkNewServerForUpdate(callback) {
        var currDir = path.join(__dirname, '..', '..');
        var backupDir = path.join(__dirname, '..', '..', 'backup');

        // create back up directory
        dibs.log.info('Creating backup directory... ');
        if (fs.existsSync(backupDir)) {
            extfs.removeSync(backupDir);
        }
        extfs.mkdirsSync(backupDir);

        async.series([
            // backup current installation
            function (cb) {
                dibs.log.info('Copying files to backup directory... ');
                backupCurrentInstallation(currDir, backupDir, cb);
            },
            // launch server on backup directory with update mode
            function (cb) {
                dibs.log.info('Launching new master server for update...');
                forkNewMasterServer(backupDir, {
                    update: currDir
                }, cb);
            }
        ], function (err) {
            if (err) {
                callback(err);
            } else {
                dibs.log.info('Terminating server after forking update server...');
                callback(new DError('MASTER014'));
            }
        });
    }


    function backupCurrentInstallation(currDir, backupDir, callback) {
        var currDir = path.join(__dirname, '..', '..');
        // copy app.js, package.json, core, lib, plugins, web-plugins, updates
        // move 'updates' directory to backup directory
        async.series([
            function (cb) {
                FileSystem.copy(path.join(currDir, 'app.js'),
                    path.join(backupDir, 'app.js'), {}, cb);
            },
            function (cb) {
                FileSystem.copy(path.join(currDir, 'package.json'),
                    path.join(backupDir, 'package.json'), {}, cb);
            },
            function (cb) {
                FileSystem.copy(path.join(currDir, 'core'),
                    path.join(backupDir, 'core'), {}, cb);
            },
            function (cb) {
                FileSystem.copy(path.join(currDir, 'lib'),
                    path.join(backupDir, 'lib'), {}, cb);
            },
            function (cb) {
                FileSystem.copy(path.join(currDir, 'plugins'),
                    path.join(backupDir, 'plugins'), {}, cb);
            },
            function (cb) {
                FileSystem.copy(path.join(currDir, 'web-plugins'),
                    path.join(backupDir, 'web-plugins'), {}, cb);
            },
            function (cb) {
                FileSystem.move(path.join(currDir, 'updates'),
                    path.join(backupDir, 'updates'), cb);
            },
            function (cb) {
                FileSystem.copy(path.join(currDir, '.info'),
                    path.join(backupDir, '.info'), {}, cb);
            }
        ], function (err) {
            callback(err);
        });
    }


    function updateAndforkNewServer(callback) {
        var installDir = path.resolve(self.startOptions.update);
        var updatesDir = path.join(__dirname, '..', '..', 'updates');
        async.series([
            // install updates to install directory
            function (cb) {
                dibs.log.info('Installing updated packages to install directory...');
                installDIBSPackages(updatesDir, installDir, cb);
            },
            // launch server on install directory
            function (cb) {
                dibs.log.info('Launching updated master server...');
                forkNewMasterServer(installDir, {}, cb);
            }
        ], function (err) {
            if (err) {
                callback(err);
            } else {
                dibs.log.info('Terminating server after forking updated master server...');
                callback(new DError('MASTER014'));
            }
        });
    }


    function downloadUpdateFromInternalRepository(pkgName, options, callback) {
        var servers = dibs.getServersByType('repo');
        if (servers.length <= 0) {
            callback(new DError('MASTER016')); return;
        }

        async.waterfall([
            function (cb) {
                dibs.log.info('Searching distributions for updates...' + pkgName);
                dibs.rpc.repo.searchDistributions({
                    repoType: 'dibs'
                }, cb);
            },
            function (dists, cb) {
                if (dists.length <= 0) {
                    cb(new DError('MASTER016')); return;
                }
                dibs.log.info('Searching snapshots for updates...' + pkgName);
                dibs.rpc.repo.searchSnapshots({
                    repoType: 'dibs',
                    distName: dists[0].name,
                    snapshotName: null
                }, function (err, snaps) {
                    if (snaps.length <= 0) {
                        cb(new DError('MASTER016'));
                        return;
                    } else {
                        cb(err, dists[0].name, snaps[0].name);
                    }
                });
            },
            function (distName, snapshotName, cb) {
                dibs.log.info('Downloading package for updates...' + pkgName + ' from ' + distName + ':' + snapshotName);
                dibs.rpc.repo.downloadRemotePackage(pkgName, {
                    repoType: 'dibs',
                    distName: distName,
                    snapshotName: snapshotName
                }, cb);
            }, function (dfsPath, cb) {
                dibs.log.info('Copying package files to updates directory...' + pkgName);
                var lPath = path.join(__dirname, '..', '..', 'updates', path.basename(dfsPath));
                dfs.getFile(lPath, dfsPath, cb);
            }], function (err) {
            callback(err);
        });
    }


    function checkUpdatesFromInternalRepository(options, callback) {
        var servers = dibs.getServersByType('repo');
        if (servers.length <= 0) {
            callback(new DError('MASTER016')); return;
        }

        async.waterfall([
            function (cb) {
                dibs.log.info('Searching distributions for checking updates...');
                dibs.rpc.repo.searchDistributions({
                    repoType: 'dibs'
                }, cb);
            },
            function (dists, cb) {
                if (dists.length <= 0) {
                    cb(new DError('MASTER016')); return;
                }

                dibs.log.info('Searching snapshots for checking updates...');
                dibs.rpc.repo.searchSnapshots({
                    repoType: 'dibs',
                    distName: dists[0].name,
                    snapshotName: null
                }, cb);
            }, function (snapshots, cb) {
                if (snapshots.length <= 0 || !snapshots[0].packages['dibs']) {
                    dibs.log.warn('No package information for dibs!');
                    cb(null, '0.0.0', []); return;
                } else {
                    dibs.log.warn('Package information found for dibs!');
                    cb(null, snapshots[0].packages['dibs'].version, []);
                }
            }], function (err, coreVersion, updates) {
            callback(err, coreVersion, updates);
        });
    }


    function relaunchMyself(callback) {
        async.series([
            function (cb) {
                self.terminate({
                    noProcessExit: true
                }, cb);
            },
            function (cb) {
                if (existsUpdates()) {
                    dibs.log.info('Updates exists!');
                    console.log('Updates exists! Updating DIBS...');
                    forkNewServerForUpdate(cb);
                } else {
                    var installDir = path.join(__dirname, '..', '..');
                    dibs.log.info('Relaunching master server...');
                    forkNewMasterServer(installDir, {}, cb);
                }
            }], function (err) {
            callback(err);
            setTimeout(function () {
                process.exit(0);
            }, 500);
        });
    }


    function stopMyself(callback) {
        async.series([
            function (cb) {
                self.terminate({}, cb);
            }], function (err) {
            callback(err);
        });
    }


    function stopSubServer(server, callback) {
        async.series([
            function (cb) {
                self.log.info('Unregistering DFS server if required... ' + server.id);
                dfs.masterIndex.unregisterServer(server.id, cb);
            },
            function (cb) {
                self.log.info('Terminating sub-server... ' + server.id);
                server.terminateServer(cb);
            },
            function (cb) {
                self.log.info('Waiting for DISCONNECTED status... ' + server.id);
                waitForServerIsDisconnected(server.id, cb);
            }
        ], function (err) {
            if (!err) {
                self.log.info('Server \'' + server.id + '\' have been stopped!');
            } else {
                self.log.error('Stopping server \'' + server.id + '\' failed!');
            }
            callback(err);
        });
    }


}

/**
 * @function createServer
 * @param {string} serverId - serverId
 * @returns {string}
 * @memberOf module:servers/master/server
 */


/**
 * @module servers/master/server
 */
