/**
 * dist-fs.js
 * Copyright (c) 2000 - 2017 Samsung Electronics Co., Ltd. All rights reserved.
 *
 * Contact:
 * Sungmin Kim <sm.art.kim@samsung.com>
 * Jonghwan Park <iwin100.park@samsung.com>
 * Kitae Kim <kt920.kim@samsung.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Contributors:
 * - S-Core Co., Ltd
**/

var fs = require('fs');
var extfs = require('fs-extra');
var path = require('path');
var async = require('async');

var dibs = require('../../core/dibs.js');
var FileServer = require('./dfs-server.js');
var DFSMasterIndex = require('./dfs-master-index.js');
var utils = require('../../lib/utils');
var DError = require('../../core/exception.js');
var FileSystem = require('../dibs.core/filesystem.js');
var Process = require('../dibs.core/process.js');

// constants
var INDEX_FILE = '.index';
var LOCAL_DIR = 'local';
var INCOMING_DIR = 'incoming';
var LOG_FILE = 'log';


/**
 * @module core/dfs/dist-fs
 */

/**
 * @constructor
 * @memberOf module:core/dfs/dist-fs
 */
function DistributedFileSystem() {
    var self = this;

    /** @type {module:core/dfs/dfs-server.FileServer} */
    this.server = new FileServer(this);
    /** @type {module:core/dfs/dfs-server.FileServer} */
    this.indexServer = null;
    this.masterIndex = null;

    //  private
    var repoPath = null;
    var maxRepoSize = 1024 * 1024 * 1024; // 1G
    var currRepoSize = 0;
    var index = {};
    var dfsLog = null;
    var fileLocks = {};
    var cacheCleanLock = false;


    /**
     * @method openRepository
     * @param {string} rPath
     * @returns {undefined}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.openRepository = function (repositoryPath, repoSize1, cleanUp, port, callback) {
        // set path & size
        repoPath = repositoryPath;
        maxRepoSize = repoSize1;

        // create logger
        dfsLog = createDFSLog(path.join(repoPath, LOG_FILE));

        // check if creating new one or loading existing one
        var indexFilePath = path.join(repoPath, INDEX_FILE);
        fs.exists(indexFilePath, function (exists) {
            if (!exists || cleanUp === true) {
                createNewRepository(function (err) {
                    // launch server
                    self.server.createServer();
                    self.server.listen(port);

                    // schedule to check local repository after 1 secs
                    dibs.thisServer.addScheduledAction('dfs-checker', {
                        period: 1000
                    }, checkLocalRepository);

                    callback(err);
                });
            } else {
                loadExistingRepository(function (err) {
                    // launch server
                    self.server.createServer();
                    self.server.listen(port);

                    // schedule to check local repository after 1 secs
                    dibs.thisServer.addScheduledAction('dfs-checker', {
                        period: 1000
                    }, checkLocalRepository);

                    callback(err);
                });
            }
        });
    };


    this.isOpened = function () {
        return ((dfsLog) ? true : false);
    };


    /**
     * @method closeRepository
     * @returns {undefined}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.closeRepository = function (callback) {
        async.series([
            // MUST BE OPENED BEFORE CLOSE
            function (cb) {
                if (repoPath !== null && index !== null) {
                    var indexFilePath = path.join(repoPath, INDEX_FILE);
                    extfs.outputJson(indexFilePath, index, cb);
                } else {
                    cb(null);
                }
            },
            // close log
            function (cb) {
                if (dfsLog) {
                    dfsLog.close();
                    dfsLog = null;
                }
                cb(null);
            }], function (err) {
            if (callback) {
                callback(err);
            }
        });
    };


    /**
     * @method cleanRepository
     * @returns {undefined}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.cleanRepository = function () {
        currRepoSize = 0;
        index = {};
    };


    /**
     * @method getRepositoryPath
     * @returns {string}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.getRepositoryPath = function () {
        return repoPath;
    };


    /**
     * @method getIncommingPath
     * @returns {string}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.getIncomingPath = function () {
        return path.join(repoPath, INCOMING_DIR);
    };


    /**
     * @method setMaxSize
     * @returns {undefined}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.setMaxSize = function (size) {
        maxRepoSize = size;
    };


    /**
     * @method setMaxSize
     * @returns {undefined}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.updateUsedTime = function (dfsPath) {
        if (index[dfsPath] !== undefined && index[dfsPath] !== null) {
            index[dfsPath].usedTime = utils.generateTimeStamp(true);
        }
    };


    /**
     * @method connect
     * @returns {undefined}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.connect = function (callback) {
        var current = dibs.thisServer;

        if (current.type === 'master' || current.type === 'agent') {
            dfsLog.info('Creating DFS master index DFS...');
            self.masterIndex = new DFSMasterIndex(self);
            self.masterIndex.registerServer(current.id, index);
            self.indexServer = dibs.getServer(current.id);
            callback(null); return;
        }

        // query index server from master
        var master = dibs.getMasterServer();
        dfsLog.info('Connecting master index server...');
        self.indexServer = master;

        connectToMasterIndexServer(function (err) {
            if (err) {
                // if index mismatched, clean up local index
                if (err.errno === 'DFS016') {
                    dfsLog.warn('Cleaning DFS local index and files...');
                    self.cleanRepository();
                    validateFileIndex(callback);
                } else {
                    self.indexServer = null;
                    callback(new DError('DFS017', {
                        sid: master.id
                    }, err));
                }
            } else {
                callback(null);
            }
        });
    };


    /**
     * @method existsFileMasterIndex
     * @param {string} rPath
     * @param {object} callback
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.existsFile = function (dfsPath, callback) {
        if (existsFileInLocal(dfsPath)) {
            callback(null, true); return;
        }

        self.indexServer.__existsFile(dfsPath, callback);
    };


    /**
     * @method addFile
     * @param {string} rPath
     * @param {string} lPath
     * @param {module:lib/utils.callback_error} callback
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.addFile = function (dfsPath, lPath, opts, callback) {
        dfsLog.info('Adding file to DFS server(path: ' + lPath + ' | dfsPath: ' + dfsPath + ')');

        // check number of arguments & their types, manipulate them
        if (typeof opts === 'function') {
            callback = opts; opts = {};
        }

        fs.exists(lPath, function (exists) {
            if (!exists) {
                callback(new DError('DFS001', {
                    path: lPath
                }), null); return;
            } else {
                opts.dfsPath = dfsPath;
                addLocalFile(lPath, opts, callback);
            }
        });
    };


    function addLocalFile(lPath, opts, callback) {
        var dfsPath = opts.dfsPath;
        // check if same file already exists
        // NOTE. if lifetime specified, we must create new path
        //   because we prevent deleting it within its lifetime
        if (opts.name && opts.size && opts.checksum && !opts.lifetime) {
            var result = searchFileInLocalRepository(dfsPath, opts.name, opts.size, opts.checksum);

            if (result) {
                return callback(null, result);
            }
        }

        var masterLock = null;
        var fInfo = null;
        var noNeedToRegister = false;
        async.waterfall([
            function (cb) {
                dfsLog.info('Reading file info...');
                readFileInfo(lPath, opts, cb);
            },
            function (info, cb) {
                fInfo = info;
                if (!dfsPath) {
                    dfsLog.info('Getting/Creating remote file path...');
                    getRemoteFilePath(info, cb);
                } else {
                    cb(null, dfsPath);
                }
            },
            function (r, cb) {
                dfsPath = r;
                dfsLog.info('Aquiring master DFS lock for adding file... ' + dfsPath);
                self.indexServer.__aquireFileLock(dibs.thisServer.id, dfsPath,
                    {
                        exclusive: true
                    }, cb);
            },
            // register file to local repository
            function (lock, cb) {
                masterLock = lock;
                if (index[dfsPath]) {
                    dfsLog.info('The file is already registered... ' + dfsPath);
                    noNeedToRegister = true;
                    if (fInfo.size !== index[dfsPath].size || fInfo.checksum !== index[dfsPath].checksum) {
                        dfsLog.warn('-File and index information are mismatched(' + dfsPath + ')');
                        dfsLog.warn('--Size: file(' + fInfo.size + ') <-> index(' + index[dfsPath].size + ')');
                        dfsLog.warn('--Checksum: file(' + fInfo.checksum + ') <-> index(' + index[dfsPath].checksum + ')');
                        cb(new DError('DFS005', {
                            size: fInfo.size,
                            hsize: index[dfsPath].size,
                            checksum: fInfo.checksum,
                            hchecksum: index[dfsPath].checksum
                        }));
                    } else {
                        if (opts.lifetime) {
                            updateLifetime(dfsPath, utils.generateTimeStamp(true), opts.lifetime);
                            dfsLog.info('-Updated file:' +
                                ' filePath(' + dfsPath + ') |' +
                                ' addedTime(' + index[dfsPath].addedTime + ') |' +
                                ' lifeTime(' + index[dfsPath].lifetime + ')');
                        }
                        cb(null);
                    }
                } else {
                    dfsLog.info('Registering file to local repository... ' + dfsPath);
                    addFileToLocalRepository(dfsPath, lPath,
                        fInfo.size, fInfo.checksum, opts.lifetime, opts.strictLifetime, cb);
                }
            },
            // update master index
            function (cb) {
                if (noNeedToRegister) {
                    cb(null);
                } else {
                    dfsLog.info('Registering file to master DFS index... ' + dfsPath);
                    self.indexServer.__registerFileIndex(dibs.thisServer.id, dfsPath, index[dfsPath], cb);
                }
            }
        ], function (err) {
            if (!err) {
                dfsLog.info('Adding file to DFS server succeeded!... ' + dfsPath);
            } else {
                dfsLog.warn('Adding file to DFS server failed!... ' + dfsPath);
            }
            dfsLog.info('-currentRepoSize: ' + currRepoSize + '/ maxRepoSize: ' + maxRepoSize);

            if (masterLock) {
                dfsLog.info('Releasing master DFS lock... ' + dfsPath);
                self.indexServer.__releaseFileLock(masterLock, function (err1) {
                    dfsLog.info('Have released master DFS lock... ' + dfsPath);
                    callback(err, dfsPath);
                });
            } else {
                callback(err, dfsPath);
            }
        });
    }


    function getRemoteFilePath(info, callback) {
        // check if it exists in local repository
        var dfsPath = getLocalFileByCheckSumAndSizeAndName(info.checksum,
            info.size, info.name);

        if (dfsPath) {
            return callback(null, dfsPath);
        }

        // if not, check remote path;
        self.indexServer.__createNewFileIndex(dibs.thisServer.id, {
            name: info.name,
            checksum: info.checksum
        }, callback);
    }


    /**
     * @method getFile
     * @param {string} lPath
     * @param {string} rPath
     * @param {module:core/base-server.BaseServer} server
     * @param {module:lib/utils.callback_error} callback
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.getFile = function (lPath, dfsPath, callback) {
        var tempFilePath = null;

        dfsLog.info('Getting file from DFS server... ' + dfsPath);
        async.waterfall([
            function (cb) {
                getFileFromRepository(dfsPath, {
                    targetPath: lPath
                }, cb);
            },
            function (targetPath, fInfo, cb) {
                if (!lPath) {
                    tempFilePath = targetPath;
                }
                fInfo.dfsPath = dfsPath;
                addLocalFile(targetPath, fInfo, function (err, dfsPath) {
                    cb(err);
                });
            }], function (err) {
            if (!err) {
                dfsLog.info('Getting file from DFS server succeeded!... ' + dfsPath);
                if (lPath) {
                    // updated used time info
                    self.updateUsedTime(dfsPath);
                }
            } else {
                dfsLog.info('Getting file from DFS server failed... ' + dfsPath);
                dfsLog.error(err);
            }
            dfsLog.info('-currentRepoSize: ' + currRepoSize + '/ maxRepoSize: ' + maxRepoSize);

            // remove incoming file if needed
            if (tempFilePath) {
                removeIncomingFile(tempFilePath, function () {
                    callback(err);
                });
            } else {
                callback(err);
            }
        });
    };


    /**
     * @method removeFile
     * @param {string} rPath
     * @param {module:lib/utils.callback_error} callback
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.removeFile = function (dfsPath, callback) {
        var masterLock = null;

        dfsLog.info('Removing file from DFS server... ' + dfsPath);

        async.waterfall([
            function (cb) {
                dfsLog.info('Aquiring master DFS lock for removing file... ' + dfsPath);
                self.indexServer.__aquireFileLock(dibs.thisServer.id, dfsPath,
                    {
                        exclusive: true
                    }, cb);
            },
            function (lock, cb) {
                dfsLog.info('Have aquired master DFS lock... ' + dfsPath);
                masterLock = lock;
                self.indexServer.__removeFileIndex(dibs.thisServer.id, dfsPath, function (err) {
                    if (!err) {
                        cb(null);
                    } else if (err.errno === 'DFS009') {
                        dfsLog.warn(err.message);
                        cb(null);
                    } else {
                        cb(err);
                    }
                });
            },
            // remove local
            function (cb) {
                removeFileFromLocalRepository(dfsPath, cb);
            }
        ], function (err) {
            if (!err) {
                dfsLog.info('Removing file from DFS server succeeded!... ' + dfsPath);
            } else {
                dfsLog.warn('Removing file from DFS server failed!... ' + dfsPath);
            }
            dfsLog.info('-currentRepoSize: ' + currRepoSize + '/ maxRepoSize: ' + maxRepoSize);

            if (masterLock) {
                dfsLog.info('Releasing master DFS lock... ' + dfsPath);
                self.indexServer.__releaseFileLock(masterLock, function (err1) {
                    dfsLog.info('Have released master DFS lock... ' + dfsPath);
                    callback(err);
                });
            } else {
                callback(err);
            }
        });
    };


    /**
     * @method getRealFilePath
     * @param {string} rPath
     * @return {string}
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.getRealFilePath = function (dfsPath) {
        return path.resolve(path.join(repoPath, LOCAL_DIR, dfsPath.split('/').join(path.sep)));
    };


    this.getFileInfo = function (dfsPath) {
        if (index[dfsPath]) {
            return index[dfsPath];
        } else {
            return null;
        }
    };


    /*
     * UTILITY FUNCTIONS
     * */


    /**
     * @method addFileToServer
     * @param {string} rPath
     * @param {string} lPath
     * @param {module:core/base-server.BaseServer} server
     * @param {module:lib/utils.callback_error} callback
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.addFileToServer = function (fileName, lPath, server, callback) {

        // check if the file exists
        fs.exists(lPath, function (exists) {
            if (!exists) {
                callback(new DError('DFS001', {
                    path: lPath
                }), null);
            } else {
                // check file exists in local repo
                self.upload(lPath, fileName, server, function (err, incomingFileName) {
                    if (err) {
                        callback(err, null);
                    } else {
                        server.__addUploadedFile(incomingFileName, null, function (err, dfsPath2) {
                            callback(err, dfsPath2);
                        });
                    }
                });
            }
        });
    };


    /**
     * @method getFileFromServer
     * @param {string} rPath
     * @param {string} lPath
     * @param {module:core/base-server.BaseServer} server
     * @param {module:lib/utils.callback_error} callback
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.getFileFromServer = function (lPath, dfsPath, server, callback) {

        // check file exists in local repo
        self.download(lPath, dfsPath, server, function (err) {
            if (err) {
                callback(new DError('DFS004', {
                    host: server.host,
                    port: server.port,
                    dfsPath: dfsPath,
                    lPath: lPath
                }, err));
            } else {
                callback(null);
            }
        });
    };



    /**
     * @method monitor
     * @param {object} opt
     * @param {module:core/dfs/dist-fs.callback_erer_masterTable} callback
     * @memberOf module:core/dfs/dist-fs.DistributedFileSystem
     */
    this.monitor = function (opt, callback) {
        if (opt.type === undefined || opt.type === null) {
            callback(new DError('DFS012'), null);
            return;
        }

        switch (opt.type) {
        case 'MASTER_INDEX':
            if (self.masterIndex !== null) {
                callback(null, self.masterIndex.getRawIndex());
            } else {
                self.indexServer.__monitorDFS(opt, callback);
            }
            break;
        case 'LOCAL_INDEX':
            callback(null, index);
            break;
        case 'LOCAL':
            callback(null, {
                maxSize: maxRepoSize,
                currSize: currRepoSize,
                index: index
            });
            break;
        case 'MASTER_LOCK':
            callback(null, masterLockMgr.getTable());
            break;
        default:
            callback(new DError('DFS013', {
                type: opt.type
            }), null);
        }
    };


    // PRIVATE METHODS
    //

    function createNewRepository(callback) {
        var localDir = path.join(repoPath, LOCAL_DIR);
        var incomingDir = path.join(repoPath, INCOMING_DIR);

        async.series([
            // create local directory
            function (cb) {
                FileSystem.createEmptyDir(localDir, cb);
            },
            // create incominig directory
            function (cb) {
                FileSystem.createEmptyDir(incomingDir, cb);
            },
            function (cb) {
                // initialize file index
                index = {};

                // create index file
                var indexFilePath = path.join(repoPath, INDEX_FILE);
                extfs.outputJson(indexFilePath, index, cb);
            }], function (err) {
            callback(err);
        });
    }


    function loadExistingRepository(callback) {
        var localDir = path.join(repoPath, LOCAL_DIR);
        var incomingDir = path.join(repoPath, INCOMING_DIR);
        var indexFilePath = path.join(repoPath, INDEX_FILE);
        var index = {};

        async.series([
            // create local directory if not exist
            function (cb) {
                FileSystem.createDirIfNotExists(localDir, cb);
            },
            // create incominig directory
            function (cb) {
                FileSystem.createEmptyDir(incomingDir, cb);
            },
            function (cb) {
                index = extfs.readJson(indexFilePath, function (err, idx) {
                    if (err) {
                        index = {};
                    } else {
                        index = idx;
                    }
                    cb(null);
                });
            },
            function (cb) {
                // set -1 to lifetime if not defined
                for (var dfsPath in index) {
                    if (!index[dfsPath].lifetime) {
                        index[dfsPath].lifetime = -1;
                    }
                    if (index[dfsPath].lifetime !== -1 && !index[dfsPath].strictLifetime) {
                        index[dfsPath].strictLifetime = false;
                    }
                }
                validateFileIndex(cb);
            },
            function (cb) {
                // get current repository size
                for (var dfsPath in index) {
                    currRepoSize += index[dfsPath].size;
                }

                // save index file
                extfs.outputJson(indexFilePath, index, cb);
            }], function (err) {
            callback(err);
        });
    }


    function validateFileIndex(callback) {
        var localDir = path.join(repoPath, LOCAL_DIR);

        async.series([
            function (cb) {
                // remove invalid index
                removeInvalidIndex(cb);
            },
            function (cb) {
                // remove unused files on 'local'
                var files = utils.readdirRecursiveSync(localDir);
                async.eachSeries(files,
                    function (filePath, cb1) {
                        var key = path.relative(localDir, filePath).split(path.sep).join('/');
                        if (index[key] === undefined) {
                            deleteFileFromRepository(key, cb1);
                        }
                    }, cb);
            }], function (err) {
            callback(err);
        });
    }


    function removeInvalidIndex(callback) {
        var localDir = path.join(repoPath, LOCAL_DIR);
        async.eachLimit(
            Object.keys(index), 10,
            function (p, cb) {
                if (index[p] === undefined || index[p] === null) {
                    delete index[p]; cb(null); return;
                }

                // check index format
                if (!index[p].path || !index[p].size || !index[p].checksum ||
                    !index[p].usedTime || !index[p].addedTime) {
                    delete index[p]; cb(null); return;
                }
                var filePath = path.join(localDir, p);

                async.series([
                    // file must be exist
                    function (cb1) {
                        fs.exists(filePath, function (exists) {
                            cb1(( (!exists) ? new Error('Invalid') : null ));
                        });
                    },
                    // check size
                    function (cb1) {
                        fs.stat(filePath, function (err, stat) {
                            if (err || !stat || index[p].size !== stat.size) {
                                cb1(new Error('Invalid'));
                            } else {
                                cb1(null);
                            }
                        });
                    },
                    // check checksum
                    function (cb1) {
                        utils.getCheckSum(filePath, function (err, checksum) {
                            if (err || index[p].checksum !== checksum) {
                                cb1(new Error('Invalid'));
                            } else {
                                cb1(null);
                            }
                        });
                    }], function (err) {
                    if (err) {
                        delete index[p];
                    }

                    cb(null);
                });

            }, function (err) {
                callback(err);
            });
    }


    // this will register local index to remote index server
    function connectToMasterIndexServer(callback) {
        var current = dibs.thisServer;

        dfsLog.info('Sending local DFS index to index server...');
        self.indexServer.__connect(current.id, index, function (err) {
            if (!err) {
                dfsLog.info('Sending local DFS index succeeded!');
            } else {
                dfsLog.warn('Sending local DFS index failed!');
            }
            callback(err);
        });
    }


    function existsFileInLocal(dfsPath) {
        return (index[dfsPath] !== undefined && index[dfsPath] !== null);
    }


    function addFileToLocalRepository(dfsPath, lPath, size, checksum, lifetime, strictLifetime, callback) {
        async.series([
            // copy file
            function (cb) {
                copyFileToRepository(lPath, dfsPath, cb);
            },
            // update index
            function (cb) {
                addFileToIndex(dfsPath, size, checksum, lifetime, strictLifetime, cb);
            }], function (err) {
            if (err) {
                rollbackCopyingFile(dfsPath, function (err1) {
                    dfsLog.warn('Adding file to local-repository failed(' + dfsPath + ')');
                    dfsLog.warn(err1);
                    callback(err);
                });
            } else {
                callback(err);
            }
        });
    }


    // NOTE. This function will be executed both on sync and async
    function removeFileFromLocalRepository(dfsPath, callback) {
        async.series([
            function (cb) {
                if (existsFileInLocal(dfsPath)) {
                    removeFileFromIndex(dfsPath);
                    cb(null);
                } else {
                    cb(new DError('DFS023', {
                        path: dfsPath
                    }));
                }
            },
            function (cb) {
                deleteFileFromRepository(dfsPath, cb);
            }], function (err) {
            callback(err);
        });
    }


    function readFileInfo(lPath, opts, callback) {
        if (opts.name && opts.size && opts.checksum) {
            callback(null, opts);
        } else {
            var fileName = path.basename(lPath);
            async.waterfall([
                function (cb) {
                    fs.stat(lPath, cb);
                },
                function (stat, cb) {
                    utils.getCheckSum(lPath, function (err, checksum) {
                        cb(err, {
                            name: fileName,
                            size: stat.size,
                            checksum: checksum
                        });
                    });
                }], function (err, info) {
                callback(err, info);
            });
        }
    }


    function getLocalFileByCheckSumAndSizeAndName(checksum, size, name) {
        for (var dfsPath in index) {
            if (index[dfsPath].checksum === checksum &&
                index[dfsPath].size === size &&
                path.basename(dfsPath) === name) {

                return dfsPath;
            }
        }

        return null;
    }


    function createNewRepositoryPath(fileName, checksum) {
        var count = 0;

        var dfsPath = [dibs.thisServer.id, (count).toString(), fileName].join('/');
        while (index[dfsPath] !== undefined) {
            dfsPath = [dibs.thisServer.id, (count++).toString(), fileName].join('/');
        }

        return dfsPath;
    }


    function getFileFromRepository(dfsPath, options, callback) {
        var masterLock = null;

        async.waterfall([
            function (cb) {
                dfsLog.info('Aquiring master DFS lock for getting file... ' + dfsPath);
                self.indexServer.__aquireFileLock(dibs.thisServer.id, dfsPath,
                    {
                        exclusive: false
                    }, cb);
            },
            function (lock, cb) {
                dfsLog.info('Have aquired master DFS lock... ' + dfsPath);
                masterLock = lock;
                if (existsFileInLocal(dfsPath)) {
                    dfsLog.info('Copying file from local repository... ' + dfsPath);
                    getFileFromLocalRepository(dfsPath, cb);
                } else {
                    dfsLog.info('Downloading file from remote repository... ' + dfsPath);
                    downloadFileFromRemoteRepository(dfsPath, cb);
                }
            },
            function (sourcePath, fInfo, cb) {
                if (options.targetPath) {
                    copyFile(sourcePath, options.targetPath, function (err) {
                        cb(err, options.targetPath, fInfo);
                    });
                } else {
                    cb(null, sourcePath, fInfo);
                }
            }], function (err, targetPath, fInfo) {
            if (masterLock !== null) {
                dfsLog.info('Releasing master DFS lock... ' + dfsPath);
                self.indexServer.__releaseFileLock(masterLock, function (err1) {
                    dfsLog.info('Have released master DFS lock... ' + dfsPath);
                    callback(err, targetPath, fInfo);
                });
            } else {
                callback(err, targetPath, fInfo);
            }
        });

    }


    function getFileFromLocalRepository(dfsPath, callback) {
        var sourcePath = self.getRealFilePath(dfsPath);
        var targetPath = path.join(repoPath, INCOMING_DIR, utils.genRandom(), dfsPath);

        copyFile(sourcePath, targetPath, function (err) {
            if (err) {
                callback(err);
            } else {
                callback(null, targetPath, {
                    name: path.basename(index[dfsPath].path),
                    size: index[dfsPath].size,
                    checksum: index[dfsPath].checksum
                });
            }
        });
    }


    function downloadFileFromRemoteRepository(dfsPath, callback) {
        var fInfo = null;

        async.waterfall([
            function (cb) {
                dfsLog.info('Getting master DFS index... ' + dfsPath);
                self.indexServer.__getFileIndex(dibs.thisServer.id, dfsPath, function (err, i) {
                    dfsLog.info('Have got master DFS index... ' + dfsPath);
                    cb(err, i);
                });
            },
            function (info, cb) {
                fInfo = info;
                fInfo.name = path.basename(dfsPath);
                dfsLog.info('Downloading file from remote servers... ' + dfsPath);
                if (info && info.servers && info.servers.length > 0) {
                    downloadFileFromRemoteServers(info.servers, dfsPath, cb);
                } else {
                    dfsLog.warn('No server information on master index!...' + dfsPath);
                    cb(new DError('DFS024', {
                        path: dfsPath
                    }), null);
                }
            }], function (err, incomingFilePath) {
            callback(err, incomingFilePath, fInfo);
        });
    }


    function downloadFileFromRemoteServers(serverIds, dfsPath, callback) {
        var incomingFilePath = null;
        var servers = serverIds.map(function (e) {
            return dibs.getServer(e);
        });
        async.detectSeries(servers,
            function (server, cb) {
                dfsLog.info('Downloading file from server \'' + server.id + '\'... ' + dfsPath);
                downloadFileFromRemoteServer(server, dfsPath, function (err, incomingPath) {
                    if (err) {
                        dfsLog.warn('Downloading file \'' + dfsPath +
                            '\' from ' + server.id + ' failed! :' + err.message);
                        cb(false);
                    } else {
                        incomingFilePath = incomingPath;
                        dfsLog.info('Downloading file from server \'' + server.id + '\' done! ' + dfsPath);
                        cb(true);
                    }
                });
            },
            function (result) {
                if (result) {
                    callback(null, incomingFilePath);
                } else {
                    callback(new DError('DFS018', {
                        dfsPath: dfsPath
                    }), null);
                }
            });
    }


    function downloadFileFromRemoteServer(rServer, dfsPath, callback) {
        var incomingFilePath = path.join(repoPath,
            INCOMING_DIR, utils.genRandom(), dfsPath);

        self.download(incomingFilePath, dfsPath, rServer, function (err, dsize) {
            if (err) {
                callback(new DError('DFS004',
                    {
                        host: rServer.host,
                        port: rServer.port,
                        dfsPath: dfsPath,
                        lPath: incomingFilePath
                    }, err), null);
            } else {
                callback(null, incomingFilePath);
            }
        });
    }


    function removeIncomingFile(filePath, callback) {
        var incomingDir = path.join(repoPath, INCOMING_DIR);
        if (incomingDir === filePath) {
            callback(null); return;
        }

        async.series([
            function (cb) {
                extfs.remove(filePath, cb);
            },
            function (cb) {
                var parentPath = path.dirname(filePath);
                fs.readdir(parentPath, function (err, files) {
                    if (err) {
                        cb(err);
                    } else {
                        if (files.length === 0) {
                            removeIncomingFile(parentPath, cb);
                        } else {
                            cb(null);
                        }
                    }
                });
            }], function (err) {
            callback(err);
        });
    }


    function copyFileToLocal(lPath, dfsPath, callback) {
        var tPath = self.getRealFilePath(dfsPath);
        copyFile(tPath, lPath, function (err) {
            if (!err) {
                // updated used time info
                self.updateUsedTime(dfsPath);
            }

            callback(err);
        });
    }


    function copyFile(srcPath, tarPath, callback) {
        var parentDir = path.dirname(tarPath);
        async.series([
            function (cb) {
                fs.exists(parentDir, function (exists) {
                    if (!exists) {
                        extfs.mkdirs(parentDir, cb);
                    } else {
                        cb(null);
                    }
                });
            },
            function (cb) {
                fs.exists(srcPath, function (exists) {
                    if (!exists) {
                        dfsLog.warn(srcPath + ' file is not exists');
                        cb(new Error(srcPath + ' file is not exists'));
                    } else {
                        FileSystem.copy(srcPath, tarPath, {
                            hardlink: true
                        }, cb);
                    }
                });
            }], function (err) {
                if (err) {
                    dfsLog.warn('[Failed] copy ' + srcPath + ' to ' + tarPath);
                } else {
                    dfsLog.info('[Succeeded] copy ' + srcPath + ' to ' + tarPath);
                }
                callback(err);
            });
    }


    function copyFileToRepository(lPath, dfsPath, callback) {
        var tPath = self.getRealFilePath(dfsPath);

        async.series([
            function (cb) {
                extfs.mkdirs(path.dirname(tPath), cb);
            },
            function (cb) {
                FileSystem.copy(lPath, tPath, {
                    hardlink: true
                }, cb);
            }], function (err) {
            callback(err);
        });
    }


    function rollbackCopyingFile(dfsPath, callback) {
        if (existsFileInLocal(dfsPath)) {
            deleteFileFromRepository(dfsPath, callback);
        } else {
            callback(null);
        }
    }


    // NOTE. This function will be executed both on sync and async
    function deleteFileFromRepository(dfsPath, callback) {
        var filePath = self.getRealFilePath(dfsPath);
        var localDir = path.join(repoPath, LOCAL_DIR);

        async.series([
            // remove file
            function (cb) {
                fs.unlink(filePath, cb);
            },
            // clean up parent directories if needed
            function (cb) {
                var skipCheck = false;

                var dirPath = path.dirname(filePath);
                var checkList = [];
                while (dirPath !== localDir) {
                    checkList.push(dirPath);
                    dirPath = path.dirname(dirPath);
                }

                async.eachSeries(checkList,
                    function (checkDir, cb1) {
                        if (skipCheck) {
                            return cb1(null);
                        }

                        fs.readdir(checkDir, function (err, files) {
                            if (files.length === 0) {
                                extfs.remove(checkDir, cb1);
                            } else {
                                skipCheck = true;
                                cb1(null);
                            }
                        });
                    }, cb);
            }], function (err) {
            callback(err);
        });
    }


    function addFileToIndex(dfsPath, size, checksum, lifetime, strictLifetime, callback) {
        var lifetime1 = (lifetime > 0 ? lifetime : -1);
        var strictLifetime1 = strictLifetime ? true : false;
        var fileInfo = {
            path: dfsPath,
            size: size,
            checksum: checksum,
            usedTime: 0,
            addedTime: utils.generateTimeStamp(true),
            lifetime: lifetime1,
            strictLifetime: strictLifetime1
        };

        index[dfsPath] = fileInfo;
        currRepoSize += size;
        callback(null);
    }


    function removeFileFromIndex(dfsPath) {
        var size = index[dfsPath].size;
        delete index[dfsPath];
        currRepoSize -= size;
    }


    function updateLifetime(dfsPath, addedTime, lifetime) {
        if (addedTime) {
            index[dfsPath].addedTime = addedTime;
        }
        index[dfsPath].lifetime = lifetime ? lifetime : -1;
    }


    // NOTE. 'dfsPath' is optional
    // NOTE. return first matched rpath
    function searchFileInLocalRepository(dfsPath, name, size, checksum) {
        if (dfsPath) {
            // verify
            if (index[dfsPath] && name === path.basename(dfsPath) &&
                size === index[dfsPath].size &&
                checksum === index[dfsPath].checksum) {

                return dfsPath;
            }
        } else {
            for (var dfsPath1 in index) {
                if (name === path.basename(dfsPath1) &&
                    size === index[dfsPath1].size &&
                    checksum === index[dfsPath1].checksum) {
                    return dfsPath1;
                }
            }
        }

        return null;
    }


    function aquireFileLock(dfsPath, isExclusive, callback) {
        dfsLog.info('Aquiring DFS local lock' +
            (isExclusive ? '(E)' : '(S)') +
            '... ' + dfsPath);
        aquireFileLockInternal(dfsPath, isExclusive, function (err) {
            dfsLog.info('Aquired DFS local lock... ' + dfsPath);
            callback(err);
        });
    }


    function aquireFileLockInternal(dfsPath, isExclusive, callback) {
        if (fileLocks[dfsPath] && (fileLocks[dfsPath].exclusive || isExclusive)) {
            setTimeout(function () {
                aquireFileLockInternal(dfsPath, isExclusive, callback);
            }, 500);
        } else {
            if (fileLocks[dfsPath]) {
                fileLocks[dfsPath].refs++;
            } else {
                fileLocks[dfsPath] = {
                    exclusive: isExclusive,
                    refs: 1
                };
            }
            callback(null);
        }
    }


    function releaseFileLock(dfsPath) {
        dfsLog.info('Released DFS local lock... ' + dfsPath);
        if (fileLocks[dfsPath]) {
            fileLocks[dfsPath].refs--;
            if (fileLocks[dfsPath].refs <= 0) {
                delete fileLocks[dfsPath];
            }
        }
    }


    function checkLocalRepository(callback) {
        // skip checking when launching server
        if (dibs.thisServer.status !== 'RUNNING') {
            return callback(null);
        }

        if (cacheCleanLock) {
            return callback(null);
        }
        cacheCleanLock = true;

        async.series([
            // handle lifetime of the files of which lifetime is ended
            function (cb) {
                handleLifetimeEnded(cb);
            },
            // if expected size is larger, remove least recently used files
            function (cb) {
                if (currRepoSize > maxRepoSize) {
                    removeLeastRecentlyUsed(cb);
                } else {
                    cb(null);
                }
            }], function (err) {
            cacheCleanLock = false;
            callback(err);
        });
    }


    function handleLifetimeEnded(callback) {

        // get the files of which life is ended.
        var files = [];
        var currTime = new Date();
        for (var dfsPath in index) {
            if (index[dfsPath].lifetime && index[dfsPath].lifetime >= 0) {
                var addedTime = utils.getDateFromTimeStamp(index[dfsPath].addedTime);
                var removeTime = new Date(addedTime.getTime() + index[dfsPath].lifetime);
                if (removeTime.getTime() - currTime.getTime() <= 0) {
                    files.push(index[dfsPath]);
                }
            }
        }

        if (files.length > 0) {
            dfsLog.warn('Trying to handle files of which lifetime is ended...');
        }

        // handle
        async.eachSeries(files,
            function (file, cb) {
                if (index[file.path].strictLifetime) {
                    self.removeFile(file.path, cb);
                } else {
                    updateLifetime(file.path, null, -1);
                    cb(null);
                }
            }, callback);
    }


    function removeLeastRecentlyUsed(callback) {

        // get array of non-unused files
        var files = [];
        for (var key in index) {
            if (index[key].usedTime !== 0) {
                files.push(index[key]);
            }
        }

        // sort by usedTime
        var sortedFiles = files.filter(function (f1) {
            return !f1.lifetime || f1.lifetime < 0;
        }).sort(function (i1, i2) {
            return i1.usedTime - i2.usedTime;
        });

        // remove files until enough
        var i = 0;
        var len = sortedFiles.length;
        if (i < len && currRepoSize > maxRepoSize) {
            dfsLog.warn('DFS cache is full! Trying to remove least recently used files');
            dfsLog.info('-currentRepoSize: ' + currRepoSize + '/ maxRepoSize: ' + maxRepoSize);
        }
        async.whilst(
            function () {
                return i < len && currRepoSize > maxRepoSize;
            },
            function (cb) {
                self.removeFile(sortedFiles[i].path, function (err) {
                    i++;
                    cb(err);
                });
            },
            function (err) {
                dfsLog.info('Removed least recently used files');
                dfsLog.info('-currentRepoSize: ' + currRepoSize + '/ maxRepoSize: ' + maxRepoSize);
                callback(err);
            });
    }

    this.getLogger = function () {
        return dfsLog;
    };


    function createDFSLog(logFilePath) {
        var logId = 'DFS';
        var logOptions = {
            filename: logFilePath,
            colorize: false
        };

        return dibs.log.open(logId, logOptions);
    }


    this.download = function (lPath, dfsPath, server, callback) {
        async.retry(5,
            function (cb, results) {
                download2(lPath, dfsPath, server, function (err) {
                    cb(err, null);
                });
            },
            function (err, results) {
                callback(err);
            });
    };


    function download2(lPath, dfsPath, server, callback) {
        var outMsg = '';
        var args = [
            path.join(__dirname, 'dfs-client.js'),
            'download', dfsPath,
            '-h', server.host,
            '-p', server.port
        ];

        if (lPath) {
            args.push('--local');
            args.push(lPath);
        }
        if (dfsLog) {
            args.push('--log');
            args.push(dfsLog.getLogFilePath());
        }

        Process.create('node', args, {}, {
            onStdout: function (line) {
                outMsg += (line + '\n');
            },
            onStderr: function (line) {
                outMsg += (line + '\n');
            },
            onExit: function (code) {
                if (code === 0) {
                    callback(null);
                } else {
                    callback(new DError('FS002', {
                        cmd: 'node dfs-client.js',
                        args: args.join(' '),
                        msg: outMsg
                    }));
                }
            }
        });
    }


    this.upload = function (lPath, fileName, server, callback) {
        var outMsg = '';

        var args = [
            path.join(__dirname, 'dfs-client.js'),
            'upload', lPath,
            '-h', server.host,
            '-p', server.port
        ];
        if (fileName) {
            args.push('--name');
            args.push(fileName);
        }
        if (dfsLog) {
            args.push('--log');
            args.push(dfsLog.getLogFilePath());
        }

        Process.create('node', args, {}, {
            onStdout: function (line) {
                outMsg += (line + '\n');
            },
            onStderr: function (line) {
                outMsg += (line + '\n');
            },
            onExit: function (code) {
                if (code === 0) {
                    var result = outMsg.split('\n')[outMsg.split('\n').length - 2];

                    callback(null, result.split(':')[1]);
                } else {
                    callback(new DError('FS002', {
                        cmd: 'node dfs-client.js',
                        args: args.join(' '),
                        msg: outMsg
                    }), null);
                }
            }
        });
    };
}


module.exports = new DistributedFileSystem();
