#!/usr/bin/env python

import sys
import os
import tempfile
import atexit
import shutil
import urllib2
import subprocess
import gzip
import re
import glob
from common.git import Git, clone_gitproject
from common.gerrit import Gerrit, get_gerrit_event, GerritError, is_ref_deleted
from common.buildservice import BuildService
from common.buildtrigger import trigger_info,trigger_next
from common.utils import sync

# prepare related global variables
workspace = os.getenv('WORKSPACE')
#basic_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/snapshots/tizen/'
basic_url = 'http://download.tizen.org/snapshots/tizen/'
gbs_meta_default_profile = os.getenv('GBS_META_DEFAULT_PROFILE')
SUPPORT_PROFILES = os.getenv('SUPPORT_PROFILES')
#REPOSITORY = os.getenv('REPOSITORY')
ARCHITECTURE = os.getenv('ARCHITECTURE')
#gbs_ref_fullbuild_root = workspace
gbs_ref_fullbuild_root = '/data/gbsfullbuild-ROOT/'
BUILD_ROOT = gbs_ref_fullbuild_root + '/GBS-ROOT/'
SRC_ROOT = gbs_ref_fullbuild_root + '/SRC-ROOT/'
LIVE_ROOT = gbs_ref_fullbuild_root + '/live/'
gbs_default_build_arg='timeout 6h gbs build --threads=16 --define "jobs 8" --define "_smp_mflags -j8" --baselibs --clean-once'
mic_default_build_arg='sudo mic --non-interactive cr auto -o '+workspace+'/mic/out '
ref_profile={}
ALL_REPO_ARCH_BUILD = os.getenv('ALL_REPO_ARCH_BUILD')
rpm_arch_type_list = 'aarch64 armv7l i586 i686 noarch vanish x86_64'
RSYNC_LIVE = os.getenv('IMG_SYNC_DEST_BASE') + '/live'

class LocalError(Exception):
    """Local error exception."""
    pass

def _update_ref_bin_index(ref_binary,build_profile):
    """ Update Reference Binary Index"""
    print "-----------------------------------------------------"
    print "Update Reference Binary Index"
    print "-----------------------------------------------------"

    # prepare separate temp directory for each build
    git_prj = 'scm/meta/obs'
    git_branch="master"
    tmpdir = tempfile.mkdtemp(prefix=workspace+'/')
    atexit.register(shutil.rmtree, tmpdir)
    prjdir = os.path.join(tmpdir, git_prj)

    # clone gerrit project to local dir
    if not clone_gitproject(git_prj, prjdir):
        print >> sys.stderr, 'Error cloning %s' %git_prj
        return 1
    mygit = Git(prjdir)
    mygit.checkout(git_branch)

    for obs_prj in ref_binary.keys():
        if obs_prj != build_profile:
            continue
        print '\nobs_prj: %s' %obs_prj
        profile = obs_prj.split(':')[-1].lower()

        if obs_prj.split(':')[-2] == 'Tizen':
            t_ver = ''
            t_ver_path = '/'
            t_branch = 'tizen'
        else:
            t_ver = obs_prj.split(':')[-2] + '-'
            t_ver_path = '/' + obs_prj.split(':')[-2] + '/'
            t_branch = 'tizen_'+obs_prj.split(':')[-2]

        basic_snapshot_url = basic_url + t_ver + profile + "/"
        ref_snapshot_number = ref_binary[obs_prj].split('ref:')[1]
        if ref_snapshot_number == 'latest':
            ref_snapshot_url = basic_snapshot_url + ref_snapshot_number + "/"
        else:
            ref_snapshot_url = basic_snapshot_url + "tizen-" + t_ver + profile + "_" + ref_snapshot_number + "/"

        if ref_snapshot_url.split('/')[-1]:
            ref_snapshot_build_id = ref_snapshot_url.split('/')[-1]
        else:
            ref_snapshot_build_id = ref_snapshot_url.split('/')[-2]

        repository = []
        arch_list= {}
        obs_meta_file = prjdir + '/' + obs_prj + '/_meta'
        lines = open(obs_meta_file).readlines()
        for line in lines:
            if line.find('repository name=') != -1:
                repo_tmp=line.split('"')[1]
                repository.append(repo_tmp)
                arch_list[repo_tmp] = []
            if line.find('<arch>') != -1:
                arch_tmp = line.split("<arch>")[1].split("</arch>")[0]
                arch_list[repo_tmp].append(arch_tmp)

        with open(obs_meta_file,"rb") as f:
            meta_text=f.read()
        for paragraph in meta_text.split("repository name"):
            if paragraph.find(ARCHITECTURE) != -1:
                repo_for_one_repo_build=paragraph.split('"')[1]
                break

        ref_profile['profile']=profile 
        ref_profile['obs_prj']=obs_prj
        ref_profile['t_ver']=t_ver
        ref_profile['t_ver_path']=t_ver_path
        ref_profile['t_branch']=t_branch
        ref_profile['basic_snapshot_url']=basic_snapshot_url
        ref_profile['ref_snapshot_number'] = ref_snapshot_number
        ref_profile['ref_snapshot_url'] = ref_snapshot_url
        ref_profile['ref_snapshot_build_id'] = ref_snapshot_build_id
        ref_profile['repository'] = repository
        ref_profile['arch_list'] = arch_list
        ref_profile['repo_for_one_repo_build'] = repo_for_one_repo_build

    print 'reference profile %s' %ref_profile


def _do_repo_init_sync(repo):
    print '----start _do_repo_init_sync-----------------------------------'

    repo_src_root=SRC_ROOT+'/'+repo
    if not os.path.exists(repo_src_root):
        os.mkdir(repo_src_root)
    os.chdir(repo_src_root)

    #add '-u' option
    repo_init_arg = ' -u ssh://review.tizen.org:29418/scm/manifest'

    #add '-b' option
    #gbs fullbuild for staing branch first. 
    #If gbs fullbuild for staging branch, update gbs meta to original branch at next trigger
#    repo_init_arg += ' -b '+ref_profile['t_branch']
    repo_init_arg += ' -b '+ref_profile['t_branch']+'_staging'

    #add '-m' option
    repo_init_arg += ' -m '+ref_profile['profile']+'_'+repo+'.xml'

    #do repo init
    cmd = 'repo init' + repo_init_arg
    print 'repo init cmd: %s' %cmd
    ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
    if ret != 0:
        update_message="repo init failed"
        if len(update_message) < 119:
            trigger_next("BUILD-MONITOR", \
                    {'bm_stage':'GBSFULLBUILD_SNAPSHOT',
                     'snapshot_name':ref_profile['ref_snapshot_build_id'],
                     'gbsfullbuild_string': update_message})

        raise LocalError('repo int failed')

    #do repo sync
    print 'do repo sync'
    cmd = 'repo sync'
    ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
    if ret != 0:
        update_message="repo sync failed"
        if len(update_message) < 119:
            trigger_next("BUILD-MONITOR", \
                    {'bm_stage':'GBSFULLBUILD_SNAPSHOT',
                     'snapshot_name':ref_profile['ref_snapshot_build_id'],
                     'gbsfullbuild_string': update_message})

        raise LocalError('repo sync failed')

def ___get_index_file_name(repo,arch):
    profile_list=os.listdir(BUILD_ROOT+'/local/repos/')
    for profile in profile_list:
        if profile.find(repo) != -1:
            profile_path=profile

    index_file=BUILD_ROOT+'/local/repos/'+profile_path+'/'+arch+'/index.html'

    return index_file

def __is_gbs_build_finished(repo,arch):
    index_file=___get_index_file_name(repo,arch)
    if os.path.exists(index_file):
        return 1

    return 0

def __is_gbs_fullbuild_result_fail(repo,arch):
    print '----start __is_gbs_fullbuild_result_fail-----------------------------------'

    index_file=___get_index_file_name(repo,arch)

    f = open(index_file,'rb')
    build_result=f.read()
    f.close()

    #summary is contents between 'Build Status Summary' and 'Build Statis Details'
    summary=build_result.split('Build Status Summary')[1].split('Build Statis Details')[0]

    total_pkg_num=summary.split('<td>')[1]
    succeeded_pkg_num=summary.split('<td>')[2]

    if total_pkg_num == succeeded_pkg_num:
        print 'GBS fullbuild succeeded'
        return 0
    else:
        print 'There are errors on gbs fullbuild'
        return 1

def _check_mount_existance():
    print '----start _check_mount_existance-----------------------------------'
    umount_list = ''

    cmd='mount'
    result=subprocess.check_output(cmd,shell=True)
    for line in result.split('\n'):
        if line.find(BUILD_ROOT.replace('//','/')) != -1:
            umount_list+=line.split('on ')[1].split(' type')[0]+' '

    if umount_list:
        print 'There exist mount which is not unmouted by previous gbs fullbuild job.\n and these lists are like below:%s' %umount_list
        cmd='sudo umount '+umount_list
        subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
    else:
        print 'No mount is left by gbsfullbuild'


def _do_repo_arch_gbs_fullbuild(repo,arch):
    print '----start _do_repo_arch_gbs_fullbuild-----------------------------------'

    #add arch
    gbs_build_arg = ' -A '+arch

    #add build root
    gbs_build_arg += ' -B '+BUILD_ROOT

    cmd = gbs_default_build_arg+gbs_build_arg+' | awk \'{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush(); }\''
    print 'gbs build argument is %s' %cmd

    retry_cnt=3
    while retry_cnt > 0:
        subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
        if __is_gbs_build_finished(repo,arch):
            print 'gbs fullbuild for repo:%s arch:%s is finished.' %(repo,arch)
            break
        else:
             print 'gbs fullbuild for repo:%s arch:%s is stucked. retrying....' %(repo,arch)

        retry_cnt -= 1


    if __is_gbs_fullbuild_result_fail(repo,arch):
        # TRIGGER NEXT BUILD-MONITOR
        update_message="GBS Fullbuild Failed"
        if len(update_message) < 119:
            trigger_next("BUILD-MONITOR", \
                    {'bm_stage':'GBSFULLBUILD_SNAPSHOT',
                     'snapshot_name':ref_profile['ref_snapshot_build_id'],
                     'gbsfullbuild_string': update_message})

        raise LocalError('There are errors on GBS fullbuild for repo:%s, arch:%s' %(repo,arch))


def do_gbs_build():
    print '----start do_gbs_build-----------------------------------'

    if not os.path.exists(gbs_ref_fullbuild_root):
        os.mkdir(gbs_ref_fullbuild_root)

    _check_mount_existance()

    if os.path.exists(BUILD_ROOT):
        cmd = 'sudo rm -rf '+BUILD_ROOT
        subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)

    if not os.path.exists(SRC_ROOT):
        os.mkdir(SRC_ROOT)

    if ALL_REPO_ARCH_BUILD == 'true':
        for repo in ref_profile['repository']:
            _do_repo_init_sync(repo)
            for arch in ref_profile['arch_list'][repo]:
                print 'OBS Project: %s, repository: %s, architecture: %s gbs fullbuild start' %(ref_profile['obs_prj'],repo,arch)
                _do_repo_arch_gbs_fullbuild(repo,arch)
    else:
        repository_for_one_repo_build=ref_profile['repo_for_one_repo_build']
        print 'Profile %s : repository for one repo build is %s'\
            %(ref_profile['obs_prj'],repository_for_one_repo_build)
        _do_repo_init_sync(repository_for_one_repo_build)
        _do_repo_arch_gbs_fullbuild(repository_for_one_repo_build,ARCHITECTURE)

    #shutil.rmtree(SRC_ROOT, ignore_errors=False, onerror=None)

def _ks_add_local_repo(remote_ks_content,repo):
    print '----start _ks_add_local_repo-----------------------------------'

    remote_ks_content=remote_ks_content.replace('@BUILD_ID@',ref_profile['ref_snapshot_build_id'])
    profile_list=os.listdir(BUILD_ROOT+'/local/repos/')
    for profile in profile_list:
        if profile.find(repo) != -1:
            profile_path=profile

    local_repo = []
    gbs_built_rpm_path=os.listdir(BUILD_ROOT+'/local/repos/'+profile_path)
    print 'gbs built rpm path: %s' %gbs_built_rpm_path
    for arch in gbs_built_rpm_path:
        local_repo.append({'baseurl':'--baseurl=file://'+BUILD_ROOT+'/local/repos/'+profile_path+'/'+arch+'/'})
        local_repo[-1]['name']='--name=local_'+arch
    print 'local repo[] = %s' %local_repo

    local_repo_line=''
    for line in remote_ks_content.split('\n'):
        if line.find('repo --name=') != -1 and line.find(ref_profile['profile']) != -1:
            print 'psk-test %s' %line
            for arg in line.split(' '):
                if arg.find('baseurl=') != -1:
                    remote_url=arg
                elif arg.find('name=') != -1:
                    remote_name=arg
            for local_repo_arch in local_repo:
                if ALL_REPO_ARCH_BUILD == 'true':
                    local_repo_line+=line.replace(remote_url,local_repo_arch['baseurl']).replace(remote_name,local_repo_arch['name'])+'\n'
                else:
                    local_repo_line+=line.replace(remote_url,local_repo_arch['baseurl']).replace(remote_name,local_repo_arch['name'])+' --priority=1\n'
            if ALL_REPO_ARCH_BUILD == 'true':
                local_ks_content=remote_ks_content.replace(line,local_repo_line)
            else:
                local_ks_content=remote_ks_content.replace(line,line+' --priority=99\n'+local_repo_line)

    return local_ks_content

def _do_mic_build(ks_path):
#    cmd = 'gbs createimage --ks-file='+ks_path
    cmd = mic_default_build_arg+ks_path
    print 'mic build command is %s' %cmd
    ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
    if ret:
        # TRIGGER NEXT BUILD-MONITOR
        update_message="Image Creation Failed"
        if len(update_message) < 119:
            trigger_next("BUILD-MONITOR", \
                    {'bm_stage':'GBSFULLBUILD_SNAPSHOT',
                     'snapshot_name':ref_profile['ref_snapshot_build_id'],
                     'gbsfullbuild_string': update_message})

        raise LocalError('Image creation error for ks:%s' %ks_path)


def do_image_creation():
    print '----start do_image_creation-----------------------------------'

    tmpdir = tempfile.mkdtemp(prefix=workspace+'/')
#    atexit.register(shutil.rmtree, tmpdir)
    ks_dir=tmpdir+'/ks_files/'
    os.mkdir(ks_dir)

    for repo in ref_profile['repository']:
        image_url=ref_profile['ref_snapshot_url']+'/builddata/images/'+repo+'/image-configurations/'
        res=urllib2.urlopen(image_url)
        for line in res.read().split('"'):
            if (ALL_REPO_ARCH_BUILD == 'true' and line.find('.ks') != -1 and line.find('</a>') == -1)\
                or (ALL_REPO_ARCH_BUILD == 'false' and line.find('.ks') != -1 and line.find('</a>') == -1 and line.find(ARCHITECTURE) != -1):
                ks_file=line
                ks_url=image_url+ks_file
                print 'ks url:%s' %ks_url
                f = open(ks_dir+ks_file,'wb')
                remote_ks_content=urllib2.urlopen(ks_url).read()
                local_ks_content=_ks_add_local_repo(remote_ks_content,repo)
                local_ks_content=local_ks_content.replace('%runscript\n\n%end','')
                print 'file content:============================================================='
                print local_ks_content
                print '=========================================================================='
                f.write(local_ks_content)
                f.close()
                _do_mic_build(ks_dir+ks_file)


def copy_build_results_to_dl_server():
    print '----start copy_build_results_to_dl_server-----------------------------------'

    #reconstruct gbs build results as similar stucture of OBS live repo
    if os.path.exists(LIVE_ROOT):
        cmd = 'sudo rm -rf '+LIVE_ROOT
        subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)

    os.mkdir(LIVE_ROOT)

    obs_liverepo=LIVE_ROOT
    sync_dest=os.path.join(RSYNC_LIVE,os.getenv('GBSFULLBUILD_DL_POSTFIX'))
    for subdir in ref_profile['obs_prj'].split(':'):
        if subdir != ref_profile['obs_prj'].split(':')[-1]:
            obs_liverepo=os.path.join(obs_liverepo,subdir+':')
            sync_dest=os.path.join(sync_dest,subdir+':')
        else:
            obs_liverepo=os.path.join(obs_liverepo,subdir)
            sync_dest=os.path.join(sync_dest,subdir)
        if not os.path.exists(obs_liverepo):
            os.mkdir(obs_liverepo)

    buildlogs_root = os.path.join(obs_liverepo,'buildlogs')
    os.mkdir(buildlogs_root)

    sync_out_dir=obs_liverepo

    profile_list=os.listdir(BUILD_ROOT+'/local/repos/')
    for profile in profile_list:
        build_result_repo_path=BUILD_ROOT+'/local/repos/'+profile+'/'
        repository=profile.split('_')[-1]
        live_repo_root=obs_liverepo+'/'+repository
        if not os.path.exists(live_repo_root):
            os.mkdir(live_repo_root)
        ###########copy arch.rpm files##############
        print '--------start copy arch.rpm files-----------------'
        for arch in rpm_arch_type_list.split(' '):
            if not os.path.exists(live_repo_root+'/'+arch):
                os.mkdir(live_repo_root+'/'+arch)
            for gbs_build_arch in os.listdir(build_result_repo_path):
                print 'copy source file dir: %s' %(build_result_repo_path+'/'+gbs_build_arch+'/RPMS/*.'+arch+'.rpm')
                print 'copy destination dir: %s' %(live_repo_root+'/'+arch+'/')
                for file_name in glob.glob(build_result_repo_path+'/'+gbs_build_arch+'/RPMS/*.'+arch+'.rpm'):
                    shutil.copy(file_name, live_repo_root+'/'+arch+'/')
            #Remove folder if there is no file in arch directory
            if len(os.walk(live_repo_root+'/'+arch).next()[2]) == 0:
                os.rmdir(live_repo_root+'/'+arch)
        ###############copy src.rpm files##############
        print '--------start copy src.rpm files-----------------'
        for gbs_build_arch in os.listdir(build_result_repo_path):
            if not os.path.exists(live_repo_root+'/src/'):
                os.mkdir(live_repo_root+'/src/')
            for file_name in glob.glob(build_result_repo_path+'/'+gbs_build_arch+'/SRPMS/*.src.rpm'):
                shutil.copy(file_name, live_repo_root+'/src/')
        ###############copy buildlog files##############
        print '--------start copy buildlog files-----------------'
        buildlogs_repository = os.path.join(buildlogs_root,repository)
        os.mkdir(buildlogs_repository)
        for gbs_build_arch in os.listdir(build_result_repo_path):
            os.mkdir(os.path.join(buildlogs_repository,gbs_build_arch))
            success_log_dest=os.path.join(buildlogs_repository,gbs_build_arch,'succeeded')
            fail_log_dest=os.path.join(buildlogs_repository,gbs_build_arch,'failed')
            os.mkdir(success_log_dest)
            os.mkdir(fail_log_dest)
            #succeeded packages
            success_log_root=os.path.join(build_result_repo_path,gbs_build_arch,'logs/success')
            fail_log_root=os.path.join(build_result_repo_path,gbs_build_arch,'logs/fail')
            for success_pkg in os.listdir(success_log_root):
                src_file=success_log_root+'/'+success_pkg+'/log.txt'
                print 'success_log_dest: %s ,success_pkg: %s' %(success_log_dest,success_pkg)
                dest_file=success_log_dest+'/'+re.findall('\D*\d*\D+\d*[-]',success_pkg)[0].rstrip('-')+'.buildlog.txt'
                shutil.copy(src_file,dest_file)
            #failed packages
            for fail_pkg in os.listdir(fail_log_root):
                src_file=fail_log_root+'/'+fail_pkg+'/log.txt'
                dest_file=fail_log_dest+'/'+re.findall('\D*\d*\D+\d*[-]',fail_pkg)[0].rstrip('-')+'.buildlog.txt'
                shutil.copy(src_file,dest_file)
            #Remove folder if there is no file in arch directory
            if len(os.walk(success_log_dest).next()[2]) == 0:
                os.rmdir(success_log_dest)
            if len(os.walk(fail_log_dest).next()[2]) == 0:
                os.rmdir(fail_log_dest)
    
    #Finally, rsync live folder to download server
    print 'rsync sync_out_dir: %s, sync_dest: %s' %(sync_out_dir, sync_dest)
    sync(sync_out_dir, sync_dest)


def main():
    """script entry point"""

    print '---[JOB STARTED]----------------------------------------'

    #event = get_gerrit_event()
    event = trigger_info(os.getenv("TRIGGER_INFO"))

    # prepare separate temp directory for each build

    if event['event_type'] != "ref-updated" or event['project'] != "scm/git-ref-mapping":
        # This is just a sanity check as ref-updated is the only event we
        # react on and it's configured in the job configuraion
        print >> sys.stderr, "Configuration error: This job can't process"\
                             "project %s! Only scm/git-ref-mapping is allowed and " \
                             "event %s! Only ref-updated events are allowed" \
                             %(event['event_type'], event['project'])
        return 1

    _check_mount_existance()
    
    git_prj = event['project']
    git_branch = event['refname']
    filename = 'git-ref-mapping.xml'
    tmpdir = tempfile.mkdtemp(prefix=workspace+'/')
    atexit.register(shutil.rmtree, tmpdir)
    prjdir = os.path.join(tmpdir, git_prj)

    # clone gerrit project to local dir
    if not clone_gitproject(git_prj, prjdir):
        print >> sys.stderr, 'Error cloning %s' %git_prj
        return 1
    mygit = Git(prjdir)
    mygit.checkout(git_branch)
    commit_msg=mygit.show(event['newrev']).split('\n')[4].replace(" ","")
    commit_log=mygit.show(event['newrev'])

    ref_binary = {}
    build_profile_list = []

    lines = open(prjdir+'/'+filename).readlines()
    for line in lines:
        if line.find('branch OBS_project') != -1:
            ref_binary[line.split('"')[1]] = line.split('"')[3]

    #Add default profile supported by gbs
    if gbs_meta_default_profile:
        for profile in gbs_meta_default_profile.split(' '):
            ref_binary[profile]=profile+':ref:latest'

    print 'Each reference snapshot numbers are like below'
    print ref_binary

    print 'ALL_REPO_ARCH_BUILD : %s' %ALL_REPO_ARCH_BUILD

    for line in commit_log.splitlines():
        if line.find('+') != -1 and line.find('branch OBS_project') != -1:
            build_profile_list.append(line.split('"')[1])

    build_profile_list=list(set(build_profile_list))
    print 'gbs fullbuild target profiles are :\n%s' %build_profile_list

    original_dir=os.getcwd()
    for build_profile in build_profile_list:
        print 'gbs fullbuild start for build_profile: %s' %build_profile

        _update_ref_bin_index(ref_binary,build_profile)

        print "snapshot_name='%s'" %ref_profile['ref_snapshot_build_id']
        if not SUPPORT_PROFILES or SUPPORT_PROFILES.find(build_profile) == -1:
            print "skip fullbuild...\nGBS reference fullbuild is triggered only for %s" %SUPPORT_PROFILES
            os.chdir(original_dir)
            gbsmeta_data = {"event": event,
                              }
            trigger_next("update-gbs-meta",gbsmeta_data)
            return 0

        do_gbs_build()
        #do_image_creation()
        copy_build_results_to_dl_server()

        os.chdir(original_dir)

        # TRIGGER NEXT GBSFULLBUILD-CREATE-SNAPSHOT
        trigger_next("gbsfullbuild-create-snapshot",\
            {'event':event,
            'obs_prj':ref_profile['obs_prj'],
            'snapshot_name':ref_profile['ref_snapshot_build_id']})


if __name__ == '__main__':
    sys.exit(main())

