#
# Copyright (C) 2010, 2011, 2012, 2013, 2014 Intel, Inc.
#
#    This program is free software; you can redistribute it and/or
#    modify it under the terms of the GNU General Public License
#    as published by the Free Software Foundation; version 2 of the License.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
#!/usr/bin/python

import sys
import os
import subprocess
import re
import shutil
import tempfile
import atexit
import urllib2
import glob
import base64
import json
from common.git import Git, clone_gitproject
from common.utils import sync
from common.buildtrigger import remote_jenkins_build_job, trigger_next
from urllib import quote_plus
from datetime import datetime

SUMMARY_LOG='summary.log'
public_basic_url = 'http://download.tizen.org/snapshots/tizen/'
public_daily_release_url = 'http://download.tizen.org/releases/daily/tizen/'

class RuntimeException(Exception):
    """Local error handler"""
    pass

class UtilsError(Exception):
    """Local error handler"""
    pass

class LocalError(Exception):
    """Local error exception."""
    pass


def gbs_conf_prefix(t_ver,profile,build_repository):
    #[general] section
    gbs_conf_text = '[general]\nfallback_to_native = true\nprofile = profile.'+t_ver+profile+'_'+build_repository+'\n\n\n'

    #Profile section
    gbs_conf_text += '################## Profile Section ##################\n\n'

    return gbs_conf_text

def gbs_remote_jenkins_build_job(data):
    encoded_data = base64.b64encode(json.dumps(data))
    remote_jenkins_build_job(os.getenv('PUBLIC_JENKINS_URL'), \
                             os.getenv('PUBLIC_JENKINS_USER'), \
                             os.getenv('PUBLIC_JENKINS_PW'), \
                             jobname='BUILD-MONITOR', \
                             data='TRIGGER_INFO=%s' % (quote_plus(encoded_data)))


def find_info_from_one_buildlog(buildlog_file,spec_file_name):
    cmd = 'tail -100 '+buildlog_file
    buildlog_tail=subprocess.check_output(cmd,shell=True)

    buildtime = ''
    built_rpm_files = []
    for each_line in buildlog_tail.split('\n'):
        if each_line.find('Wrote: /home/abuild/rpmbuild/RPMS/') != -1 and each_line.find('-debuginfo-') == -1 and each_line.find('-debugsource-') == -1:
            rpm_file_name=each_line.split('/')[-1]
            built_rpm_files.append(rpm_file_name)
        if each_line.find('finished "build '+spec_file_name+'.spec') != -1:
            buildtime = each_line.split('[')[1].split('s]')[0].replace(' ','')

    print "spec_file_name: %s buildtime: %s built_rpm_files: %s" %(spec_file_name,buildtime,built_rpm_files)
    return (buildtime,built_rpm_files)

def find_spec_name_from_build_log(buildlog_file):
    cmd = 'tail -100 '+buildlog_file
    buildlog_tail=subprocess.check_output(cmd,shell=True)

    spec_name = ''
    for each_line in buildlog_tail.split('\n'):
        if each_line.find('finished "build ') != -1 or each_line.find('failed "build ') != -1:
            spec_file = each_line.split('"build ')[1].split('.spec')[0]
            return spec_file

def replace_predefined_val_from_file_content_in_name(name,file_contents):
    print 'replace_predefined_val_from_file_content_in_name for name: %s' %name

    def_list=name.split('%{')[1:]
    for each_def in def_list:
        each_def=each_def.split('}')[0]
        each_def_val=file_contents.split(each_def)[1].split('\n')[0].replace('"','').replace("'",'').replace(' ','')
        name = name.replace('%{'+each_def+'}',each_def_val)

    return name

class GBSBuild:
    """A class which supports with statement"""

    def __init__(self, gbsbuild_tag, obs_prj, basic_url, snapshot_num, trigger_category, new_pkg_list, rm_pkg_list, rm_from_img_list, add_to_img_list, gbsbuild_workspace):
        """ Initialize GBSBuild class"""
        print "-----------------------------------------------------"
        print "Initialize GBS Build Class"
        print "-----------------------------------------------------"

        workspace = os.getenv('WORKSPACE')

        # prepare separate temp directory for each build
        git_prj = 'scm/meta/obs'
        git_branch="master"
        tmpdir = tempfile.mkdtemp(prefix=workspace+'/')
        atexit.register(shutil.rmtree, tmpdir)
        prjdir = os.path.join(tmpdir, git_prj)

        # clone gerrit project to local dir
        if not clone_gitproject(git_prj, prjdir):
            print >> sys.stderr, 'Error cloning %s' %git_prj
            return
        mygit = Git(prjdir)
        mygit.checkout(git_branch)

        if len(re.findall('\D', obs_prj.split(':')[1].replace('.',''))) != 0:
            t_ver = ''
            t_ver_path = '/'
            t_branch = 'tizen'
            profile_array = obs_prj.split(':')[1:]
        else:
            t_ver = obs_prj.split(':')[1] + '-'
            t_ver_path = '/' + obs_prj.split(':')[1] + '/'
            t_branch = 'tizen_'+obs_prj.split(':')[-2]
            profile_array = obs_prj.split(':')[2:]
        
        #find profile name
        i=0
        profile = ''
        while i < len(profile_array):
            profile += profile_array[i].lower() + '-'
            i += 1
        profile = profile.rstrip('-')

        basic_snapshot_url = basic_url + t_ver + profile + "/"
        if snapshot_num == 'latest':
            snapshot_url = basic_snapshot_url + snapshot_num + "/"
        else:
            snapshot_url = basic_snapshot_url + "tizen-" + t_ver + profile + "_" + snapshot_num + "/"

        if snapshot_url.split('/')[-1]:
            snapshot_build_id = snapshot_url.split('/')[-1]
        else:
            snapshot_build_id = snapshot_url.split('/')[-2]

        repository = []
        arch_list= {}
        path_prj_list= {}
        disable_repo_arch = []
        obs_meta_file = prjdir + '/' + obs_prj + '/_meta'
        lines = open(obs_meta_file).readlines()
        for line in lines:
            if line.find('<disable arch=') != -1:
                disable_repo=line.split('repository="')[1].split('"')[0]
                disable_arch=line.split('arch="')[1].split('"')[0]
                disable_repo_arch.append(disable_repo+"##"+disable_arch)
            if line.find('repository name=') != -1:
                repo_tmp=line.split('"')[1]
                repository.append(repo_tmp)
                arch_list[repo_tmp] = []
                path_prj_list[repo_tmp] = []
            if line.find('<arch>') != -1:
                arch_tmp = line.split("<arch>")[1].split("</arch>")[0]
                if repo_tmp+"##"+arch_tmp not in disable_repo_arch:
                    arch_list[repo_tmp].append(arch_tmp)
            if line.find('path project=') != -1:
                path_prj_arch = line.split('"')[1]+'##'+line.split('"')[3]
                path_prj_list[repo_tmp].append(path_prj_arch)

        #handle for _aggregate file
        aggregate_list = []
        for root, dirs, files in os.walk(prjdir+'/'+obs_prj):
            for each_file in files:
                if each_file == "_aggregate":
                    aggregate_file=os.path.join(root,each_file)
                    lines = open(aggregate_file).readlines()
                    for line in lines:
                        if line.find('<binary>') != -1:
                            binary_rpm = line.split('<binary>')[1].split('</binary>')[0]
                            aggregate_list.append({'binary_rpm':binary_rpm})
                            aggregate_list[-1]['repository_source_target_list'] = []
                        if line.find('<repository source') != -1:
                            src_repo = line.split('<repository source="')[1].split('"')[0]
                            target_repo = line.split('target="')[1].split('"')[0]
                            aggregate_list[-1]['repository_source_target_list'].append(src_repo+'##'+target_repo)

        prj_src_root = os.path.join(gbsbuild_workspace,'SRC-ROOT',obs_prj)
        build_root = os.path.join(gbsbuild_workspace,'GBS-ROOT')
        live_root = os.path.join(gbsbuild_workspace,'live')

        if trigger_category.lower() == 'dependency build':
            is_fullbuild = 'false'
        else:
            is_fullbuild = 'true'

        build_pkg_list = {}
        if new_pkg_list:
            item_is_git_name = 'true'
            for item in new_pkg_list.split(' '):
                if item_is_git_name == 'true':
                    git_name = item
                    item_is_git_name = 'false'
                else:
                    commit_id = item
                    item_is_git_name = 'true'
                    build_pkg_list[git_name] = commit_id

        rm_pkg_list_array = []
        if rm_pkg_list:
            for git_name in rm_pkg_list.split(' '):
                rm_pkg_list_array.append(git_name)

        self.profile = profile
        self.obs_prj = obs_prj
        self.t_ver = t_ver
        self.t_ver_path = t_ver_path
        self.t_branch = t_branch
        self.basic_snapshot_url = basic_snapshot_url
        self.snapshot_num = snapshot_num
        self.snapshot_url = snapshot_url
        self.snapshot_build_id = snapshot_build_id
        self.repository = repository
        self.arch_list = arch_list
        self.path_prj_list = path_prj_list
        self.gbsbuild_workspace = gbsbuild_workspace
        self.prj_src_root = prj_src_root
        self.build_root = build_root
        self.live_root = live_root
        self.basic_url = basic_url
        self.is_fullbuild = is_fullbuild
        self.build_pkg_list = build_pkg_list
        self.rm_pkg_list_array = rm_pkg_list_array
        self.gbsbuild_tag = gbsbuild_tag
        self.trigger_category = trigger_category
        self.aggregate_list = aggregate_list
        self.rm_from_img_list = rm_from_img_list
        self.add_to_img_list = add_to_img_list

        print 'profile: %s,obs_prj: %s,t_ver: %s,t_ver_path: %s,t_branch: %s,\
               basic_snapshot_url: %s,snapshot_num: %s,snapshot_url: %s,snapshot_build_id: %s,\
               repository: %s,arch_list: %s, path_prj_list: %s,prj_src_root: %s,\
               build_root: %s, basic_url: %s, is_fullbuild: %s, build_pkg_list: %s,\
               trigger_category: %s, gbsbuild_workspace: %s, aggregate_list: %s,\
               rm_from_img_list: %s, add_to_img_list: %s'\
                %(profile,obs_prj,t_ver,t_ver_path,t_branch,basic_snapshot_url,snapshot_num,\
                 snapshot_url,snapshot_build_id,repository,arch_list,path_prj_list,prj_src_root,\
                 build_root,basic_url,is_fullbuild,build_pkg_list,trigger_category,\
                 gbsbuild_workspace,aggregate_list,rm_from_img_list,add_to_img_list) 

    def convert_gbsbuild_to_dictionary(self):
        dic = {}

        dic['profile'] = self.profile
        dic['obs_prj'] = self.obs_prj
        dic['t_ver'] = self.t_ver
        dic['t_ver_path'] = self.t_ver_path
        dic['t_branch'] = self.t_branch
        dic['basic_snapshot_url'] = self.basic_snapshot_url
        dic['snapshot_num'] = self.snapshot_num
        dic['snapshot_url'] = self.snapshot_url
        dic['snapshot_build_id'] = self.snapshot_build_id
        dic['repository'] = self.repository
        dic['arch_list'] = self.arch_list
        dic['path_prj_list'] = self.path_prj_list
        dic['gbsbuild_workspace'] = self.gbsbuild_workspace
        dic['prj_src_root'] = self.prj_src_root
        dic['build_root'] = self.build_root
        dic['live_root'] = self.live_root
        dic['basic_url'] = self.basic_url
        dic['is_fullbuild'] = self.is_fullbuild
        dic['build_pkg_list'] = self.build_pkg_list
        dic['rm_pkg_list_array'] = self.rm_pkg_list_array
        dic['gbsbuild_tag'] = self.gbsbuild_tag
        dic['trigger_category'] = self.trigger_category
        dic['aggregate_list'] = self.aggregate_list
        dic['rm_from_img_list'] = self.rm_from_img_list
        dic['add_to_img_list'] = self.add_to_img_list
        return dic

    def get_build_pkg_data(self):
        print '----start get_build_pkg_data-----------------------------------'

        build_pkg_data = []
        for git_repository in self.build_pkg_list:
            git_commit_id = self.build_pkg_list[git_repository]
            tmpdir = tempfile.mkdtemp(prefix=os.getenv('WORKSPACE')+'/')
            atexit.register(shutil.rmtree, tmpdir)
            prjdir = os.path.join(tmpdir, git_repository)
            if not clone_gitproject(git_repository, prjdir):
                print >> sys.stderr, 'Error cloning %s' %git_repository
                return 1
            mygit = Git(prjdir)
            mygit.checkout(git_commit_id)
            git_committer =  mygit.show(git_commit_id).split('\n')[1].split(':')[1].lstrip(' ')
            git_commit_message = mygit.show(git_commit_id).split('\n')[4]

            build_pkg_data.append({'git_repository':git_repository})
            build_pkg_data[-1]['git_commit_id'] = git_commit_id
            build_pkg_data[-1]['git_committer'] = git_committer
            build_pkg_data[-1]['git_commit_message'] = git_commit_message

            build_pkg_data[-1]['is_remove'] = 0

        print 'build_pkg_data info for gbsbuld: %s' %build_pkg_data
        return build_pkg_data
            

    def ___build_pkg_list_apply(self,manifest_content):
        print '----start ___build_pkg_list_apply-----------------------------------'

        build_pkg_list_new = {}
        build_pkg_list_replace = {}

        # build_pkg_list_new : git_name & commit_id for new packages
        # build_pkg_list_replace : git_name & commit_id for already existing packages --> commit id has to be replaced
        for git_name in self.build_pkg_list.keys():
            if manifest_content.find(git_name) != -1:
                build_pkg_list_replace[git_name] = self.build_pkg_list[git_name]
            else:
                build_pkg_list_new[git_name] = self.build_pkg_list[git_name]

        # for build_pkg_list_replace, line which contains git_name has to be replaced to contain new commit id
        replaced_manifest_content = ''
        for each_line in manifest_content.split('\n'):
            for git_name in build_pkg_list_replace.keys():
#                if each_line.find(git_name) != -1:
                if each_line.find('<project name="'+git_name+'"') != -1:
                    each_line = '  <project name="%s" path="%s" revision="%s"/>'\
                                     %(git_name,git_name,build_pkg_list_replace[git_name])
                    break

# rm_pkg will be removed after repo sync
#            rm_pkg_found = 'false'
#            for git_name in self.rm_pkg_list_array:
#                if each_line.find(git_name) != -1:
#                    rm_pkg_found = 'true'
#                    break
            #if rm package is found in manifest file, don't perform adding each_line to manifest file
#            if rm_pkg_found == 'true':
#                continue

            if each_line.find('</manifest>') != -1:
                continue

            replaced_manifest_content += each_line+'\n'

        # for build_pkg_list_new, line has to be added newly
        for git_name in build_pkg_list_new.keys():
            replaced_manifest_content += '  <project name="%s" path="%s" revision="%s"/>\n'\
                            %(git_name,git_name,build_pkg_list_new[git_name])

        replaced_manifest_content += '</manifest>'

        return replaced_manifest_content


    def __manifest_replace(self,repo):
        print '----start __manifest_replace-----------------------------------'
        manifest_file = self.prj_src_root+'/'+repo+'/.repo/manifests/'+self.profile+'/'+repo+'/projects.xml'
        snapshot_manifest_url = self.snapshot_url+'/builddata/manifest/'+self.snapshot_build_id+'_'+repo+'.xml'
        print 'snapshot_manifest_url : %s' %snapshot_manifest_url
        res = urllib2.urlopen(snapshot_manifest_url)
        snapshot_manifest_content = res.read()
        new_snapshot_manifest_content = self.___build_pkg_list_apply(snapshot_manifest_content)
        f = open(manifest_file,'wb')
        f.write(new_snapshot_manifest_content)
        f.close()


    def _get_base_repo_in_gbs_conf(self,path_prj_arch,repo):
        """ Get base repo url from obs project & repository"""
        print "-------_get_base_repo_in_gbs_conf start-----------------"

        daily_release_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/releases/daily/tizen/'
        basic_url = self.basic_url

        base_repo = {}
        obs_prj=path_prj_arch.split('##')[0]
        path_arch=path_prj_arch.split('##')[1]
        path_prj=obs_prj.replace('Tizen:','').replace(':','-').lower()

        if obs_prj.find(':ref:') != -1:
            # Case of specific base snapshot number used for path project is specified in obs meta
            ref_build_id=obs_prj.split(':ref:')[1]
            obs_prj=obs_prj.split(':ref:')[0]
            snapshot_id=obs_prj.replace('Tizen:','').replace(':','-').lower()
            temp_id = '/'+snapshot_id+'/tizen-'+snapshot_id+'_'+ref_build_id+'/repos/'+path_arch
            url_candidates = []
            url_candidates.append(basic_url+temp_id)
            url_candidates.append(daily_release_url+temp_id)
            url_candidates.append(public_basic_url+temp_id)
            url_candidates.append(public_daily_release_url+temp_id)
            for url in url_candidates:
                try:
                    urllib2.urlopen(url)
                except:
                    continue
                else:
                    base_url = url
                    break
        else:
            # Case of Tizen:Base / Tizen:4.0:Base / Tizen 3.0:Base
            # Must find base snapshot id which is lower than snapshot_num
            snapshot_id=path_prj
            res = urllib2.urlopen(basic_url + '/'+snapshot_id+'/')
            base_url_list = res.read()
            prev_build_id = '0'
            base_build_id = '0'
            for line in base_url_list.split('\n'):
                if line.find('<a href="') != -1 and line.find('tizen-'+snapshot_id) != -1:
                    build_id=line.split('"')[1].replace('tizen-'+snapshot_id+'_','').rstrip('/')
                    if float(build_id) >= float(self.snapshot_num):
                        if prev_build_id == '0':
                            base_build_id = build_id
                        else:
                            base_build_id = prev_build_id
                        break
                    else:
                        base_build_id = build_id
                        prev_build_id = build_id
            base_url = basic_url+'/'+snapshot_id+'/tizen-'+snapshot_id+'_'+base_build_id+'/repos/'+path_arch

        print 'obs_prj: %s, snapshot_id: %s' %(obs_prj,snapshot_id)
        print 'path_prj: %s, path_arch: %s, base_url: %s' %(path_prj,path_arch,base_url)

        base_repo['repo_name']='repo.'+self.t_ver+self.profile+'_'+repo+'__'+path_prj+'_'+path_arch
        base_repo['debug_repo_name']='repo.'+self.t_ver+self.profile+'_'+repo+'__'+path_prj+'_'+path_arch+'_debug'
        base_repo['repo_url']=base_url+'/packages/'
        base_repo['debug_repo_url']=base_url+'/debug/'

        print 'base_repo: %s' %base_repo

        return base_repo


    def ___get_profile_repo_section_for_one_profile_repo(self,repo):
        #profile section
        profile = self.profile
        t_ver = self.t_ver
        t_ver_path = self.t_ver_path
        snapshot_url = self.snapshot_url

        gbs_conf_text = '############# '+ t_ver+profile+' #############\n'
        gbs_conf_text += '[profile.'+t_ver+profile+'_'+repo+']\n'
        if self.is_fullbuild == 'true':
            gbs_conf_text += 'buildconf=./scm/meta/build-config'\
                                  +t_ver_path+profile+'/'+repo+'_build.conf\n'
        gbs_conf_text += 'repos = '
        for path_prj_arch in self.path_prj_list[repo]:
            base_repo = self._get_base_repo_in_gbs_conf(path_prj_arch,repo)
            gbs_conf_text += base_repo['repo_name']+','+base_repo['debug_repo_name']+','
            if self.is_fullbuild == 'false':
                gbs_conf_text += 'repo.'+t_ver+profile+'_'+repo+','
                gbs_conf_text += 'repo.'+t_ver+profile+'_'+repo+'_debug,'
        gbs_conf_text = gbs_conf_text[:-1]

        gbs_conf_text += '\n\n'

        ####repo section####
        gbs_conf_text += '\n####################### Repo Section#######################\n'

        #base repo
        gbs_conf_text += '\n############# base #############\n'

        for path_prj_arch in self.path_prj_list[repo]:
            base_repo = self._get_base_repo_in_gbs_conf(path_prj_arch,repo)
            gbs_conf_text += '['+base_repo['repo_name']+']\n'
            gbs_conf_text += 'url = '+base_repo['repo_url']+'\n'
            gbs_conf_text += '['+base_repo['debug_repo_name']+']\n'
            gbs_conf_text += 'url = '+base_repo['debug_repo_url']+'\n\n'

        #profile repo
        gbs_conf_text += '\n############# '+ t_ver+profile+' #############\n'
        gbs_conf_text += '[repo.'+t_ver+profile+'_'+repo+']\n'
        gbs_conf_text += 'url = '+snapshot_url+'repos/'+repo+'/packages/\n'
        gbs_conf_text += '[repo.'+t_ver+profile+'_'+repo+'_debug]\n'
        gbs_conf_text += 'url = '+snapshot_url+'repos/'+repo+'/debug/\n\n'

        return gbs_conf_text


    def __update_gbs_conf_for_one_profile_repo(self,repo):
        print '----start __update_gbs_conf-----------------------------------'

        gbs_conf_file = self.prj_src_root+'/'+repo+'/.gbs.conf'
        if os.path.exists(gbs_conf_file):
            os.remove(gbs_conf_file)

        gbs_conf_text = gbs_conf_prefix(self.t_ver,self.profile,repo)
        gbs_conf_text += self.___get_profile_repo_section_for_one_profile_repo(repo)

        print gbs_conf_text

        f = open(gbs_conf_file,'wb')
        f.write(gbs_conf_text)
        f.close()

    def __rm_rpm_from_meta_git(self,repo):
        print '----start __rm_rpm_from_meta_git-----------------------------------'
        src_path = os.path.join(self.prj_src_root,repo)
        rm_rpms_from_img = []

        for git_name in self.rm_from_img_list.split(' '):
            for each_file in os.listdir(os.path.join(src_path,git_name,'packaging')):
                if each_file.endswith('.spec'):
                    spec_file=os.path.join(src_path,git_name,'packaging',each_file)
                    with open(spec_file,"rb") as f:
                        file_contents = f.read()
                    for line in file_contents.split('\n'):
                        if line.find('Name:') != -1:
                            orig_rpm_name = line.split('Name:')[1].replace(' ','').replace('\t','')
                            #When pre-defined value is used in 'Name' filed
                            if orig_rpm_name.find('%{') != -1:
                                orig_rpm_name = replace_predefined_val_from_file_content_in_name(orig_rpm_name,file_contents)
                            rm_rpms_from_img.append(orig_rpm_name)
                        if line.find('%package') != -1:
                            sub_pkg_post_name = line.split('%package')[1].replace(' ','').replace('\t','')
                            if sub_pkg_post_name.find('%{') != -1:
                                sub_pkg_post_name = replace_predefined_val_from_file_content_in_name(sub_pkg_post_name,file_contents)
                            rm_rpms_from_img.append(orig_rpm_name+'-'+sub_pkg_post_name)
        print 'Remove rpm list from image: %s' %rm_rpms_from_img

        cmd = 'find '+src_path+' -name patterns | grep meta'
        result = subprocess.check_output(cmd,shell=True)

        for each_dir in result.split('\n'):
            print 'meta dir name: %s' %each_dir
            if not os.path.exists(each_dir):
                continue
            for each_file in os.listdir(each_dir):
                if each_file.endswith('.yaml'):
                    each_file_path=os.path.join(src_path,each_dir,each_file)
                    with open(each_file_path,"rb") as f:
                        file_content=f.read()
                    file_content_orig=file_content
                    for rpm_name in rm_rpms_from_img:
                        file_content=file_content.replace('- '+rpm_name+'\n','')
                    if file_content != file_content_orig:
                        print "file: %s is modified in the process of remove rpms from image" %each_file_path
                        with open(each_file_path,"wb") as f2:
                            f2.write(each_file_path)


    def __add_rpm_to_meta_git(self,repo):
        print '----start __add_rpm_to_meta_git-----------------------------------'
        src_path = os.path.join(self.prj_src_root,repo)

        for add_pkg in self.add_to_img_list:
            #find .yaml file path 
            cmd = 'find '+src_path+' -name "'+add_pkg['pattern_name'] +'.yaml" | grep patterns'
            result = subprocess.check_output(cmd,shell=True)
            meta_file_path=result.split('\n')[0]
            print '%s is added to %s' %(add_pkg['rpm_name'], meta_file_path)
            #add rpm name to *.yaml file
            with open(meta_file_path,"rb") as f:
                file_content=f.read()
            if not file_content.endswith('\n'):
                file_content += '\n'
            file_content = file_content + '- ' +add_pkg['rpm_name']
            with open(meta_file_path,"wb") as f2:
                f2.write(file_content)


    def __rm_git_from_build_src(self,repo):
        print '----start __rm_git_from_build_src-----------------------------------'
        src_path = os.path.join(self.prj_src_root,repo)

        for rm_git in self.rm_pkg_list_array:
            cmd = 'sudo rm -rf '+os.path.join(src_path,rm_git)
            print 'Remove git %s from build source with cmd: %s' %(rm_git,cmd)
            subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)


    def _get_gbs_build_result_status(self,repo,arch):
        print '----start _get_gbs_build_result_status-----------------------------------'

        profile_list=os.listdir(self.build_root+'/local/repos/')
        for profile in profile_list:
            if profile.find(repo.replace('-','_')) != -1:
                profile_path=profile

        index_file=self.build_root+'/local/repos/'+profile_path+'/'+arch+'/index.html'

        f = open(index_file,'rb')
        build_result=f.read()
        f.close()

        #summary is contents between 'Build Status Summary' and 'Build Statis Details'
        summary=build_result.split('Build Status Summary')[1].split('Build Statis Details')[0]

        total_pkg_num=summary.split('<td>')[1].split('</td>')[0]
        succeeded_pkg_num=summary.split('<td>')[2].split('</td>')[0]
        export_err_pkg_num=summary.split('<td>')[3].split('</td>')[0]
        expansion_err_pkg_num=summary.split('<td>')[4].split('</td>')[0]
        build_err_pkg_num=summary.split('<td>')[5].split('</td>')[0]

        if total_pkg_num == succeeded_pkg_num:
            status = 'S'
            status_reason = 'Succeeded'
        elif export_err_pkg_num != '0':
            status = 'F'
            status_reason = 'Export Error'
        elif expansion_err_pkg_num != '0':
            status = 'F'
            status_reason = 'Expansion Error'
        else:
            status = 'F'
            status_reason = 'Build Error'

        print 'GBS Build status: %s, status_reason: %s' %(status,status_reason)

        summary_log_file=self.build_root+'/local/repos/'+profile_path+'/'+arch+'/'+SUMMARY_LOG
        summary_log_content='GBS Build status: "%s", status_reason: "%s"' %(status,status_reason)
        f = open(summary_log_file,'wb')
        f.write(summary_log_content)
        f.close()

        return (status,status_reason)


    def _do_repo_init_sync(self,repo):
        print '----start _do_repo_init_sync-----------------------------------'
        repo_src_root=self.prj_src_root+'/'+repo

        if not os.path.exists(self.prj_src_root):
            os.mkdir(self.prj_src_root)

        if not os.path.exists(repo_src_root):
            os.mkdir(repo_src_root)
        os.chdir(repo_src_root)

        if os.path.exists(os.path.join(repo_src_root,'.repo')):
            repo_init_clear_files=os.path.join(repo_src_root,'.repo/manifests*')
            cmd = 'sudo rm -rf '+repo_init_clear_files
            subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)

        # rm meta git, because they can have un-commited changes from previous gbs build
        cmd = 'find '+repo_src_root+' -name patterns | grep meta'
        try:
            result = subprocess.check_output(cmd,shell=True)
            result = result.replace('/patterns\n','\n')
            for meta_git in result.split('\n'):
                if meta_git:
                    print 'remove meta_git: %s for next gbs build' %meta_git
                    cmd = 'sudo rm -rf '+meta_git
                    subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
        except:
            print "There is no meta git"

        #add '-u' option
        repo_init_arg = ' -u ssh://%s:%s/scm/manifest' %(os.getenv('GERRIT_HOSTNAME_EXTERNAL'), 
#        repo_init_arg = ' -u ssh://%s:%s/scm/manifest' %(os.getenv('PUBLIC_GERRIT_HOSTNAME'),
                                                         os.getenv('GERRIT_SSHPORT'))
        #add '-b' option
        if self.gbsbuild_tag.startswith('REF'):
            repo_init_arg += ' -b '+self.t_branch+'_staging'
        else:
            repo_init_arg += ' -b '+self.t_branch
        #add '-m' option
        repo_init_arg += ' -m '+self.profile+'_'+repo+'.xml'
        
        cmd = 'repo init' + repo_init_arg

        #do repo init
        cmd = 'repo init' + repo_init_arg
        print 'repo init cmd: %s' %cmd
        ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
        if ret != 0:
            print 'repo int failed!!!!!'
            repo_init_sync_status = 'F'
            status_reason = 'repo init failed'
            return (repo_init_sync_status, status_reason)

        self.__manifest_replace(repo)

        #do repo sync
        print 'do repo sync'
        cmd = 'repo sync'
        ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)
        if ret != 0:
            print 'repo sync failed!!!!!'
            repo_init_sync_status = 'F'
            status_reason = 'repo sync failed'
            return (repo_init_sync_status, status_reason)

        self.__update_gbs_conf_for_one_profile_repo(repo)

        if self.rm_from_img_list:
            self.__rm_rpm_from_meta_git(repo)
        if self.add_to_img_list:
            self.__add_rpm_to_meta_git(repo)
        if self.rm_pkg_list_array:
            self.__rm_git_from_build_src(repo)

        repo_init_sync_status = 'S'
        status_reason = 'Succeeded'

        return (repo_init_sync_status, status_reason)


    def __find_binary_list_arg(self,repo):
        """ Find argument of --binary-list : which is same as to find spec file name in build_pkg_list"""
        print '----start __find_binary_list_arg-----------------------------------'
        binary_list = ''
        for git_name in self.build_pkg_list.keys():
            for each_file in os.listdir(os.path.join(self.prj_src_root,repo,git_name,'packaging')):
                if each_file.endswith('.spec'):
                    binary_list += each_file.replace('.spec','')+','
        binary_list = binary_list.rstrip(',')

        print 'build spec file list: %s' %binary_list
        return binary_list


    def _do_repo_arch_gbs_fullbuild(self,repo,arch,gbs_build_arg=None):
        print '----start _do_repo_arch_gbs_fullbuild-----------------------------------'

        if not gbs_build_arg:
            gbs_default_build_arg='timeout 6h gbs build --threads=16 --define "jobs 8" --define "_smp_mflags -j8" --baselibs --clean-once --include-all'
        else:
            gbs_default_build_arg='timeout 6h gbs build '+gbs_build_arg

        #add arch
        gbs_build_arg = ' -A '+arch

        #add build root
        gbs_build_arg += ' -B '+self.build_root

        #for dependency build, add --rdeps and --binary-list
        if self.is_fullbuild == 'false':
            binary_list_arg = self.__find_binary_list_arg(repo)
            gbs_build_arg += ' --rdeps --binary-list='+binary_list_arg

        cmd = gbs_default_build_arg+gbs_build_arg\
              +' | awk \'{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush(); }\''
        print 'gbs build argument is %s' %cmd

        subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)

        (gbs_build_status,gbs_build_status_reason) = self._get_gbs_build_result_status(repo,arch)
        print "GBS Fullbuild result: %s" %gbs_build_status_reason

        return (gbs_build_status,gbs_build_status_reason)


    def do_gbs_build(self):
        print '----start do_gbs_build-----------------------------------'


        if not os.path.exists(self.prj_src_root):
            os.mkdir(self.prj_src_root)

        for repo in self.repository:
#        for repo in ['standard']:
            self._do_repo_init_sync(repo)
            for arch in self.arch_list[repo]:
#            for arch in ['armv7l']:
                print 'OBS Project: %s, repository: %s, architecture: %s gbs build start'\
                       %(self.obs_prj,repo,arch)
                self._do_repo_arch_gbs_fullbuild(repo,arch)


    def copy_build_results_to_dl_server(self,repository):
        print '----start copy_build_results_to_dl_server-----------------------------------'

        path_live_repo_base = os.getenv('PATH_LIVE_REPO_BASE').rstrip('/')
        RSYNC_LIVE = os.getenv('IMG_SYNC_DEST_BASE')+'/'+os.path.basename(path_live_repo_base)
        rpm_arch_type_list = 'aarch64 armv7l i586 i686 noarch vanish x86_64'

        live_root = self.live_root
        #reconstruct gbs build results as similar stucture of OBS live repo
        if os.path.exists(live_root):
            cmd = 'sudo rm -rf '+live_root
            subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True)

        os.mkdir(live_root)

        obs_liverepo=live_root
        sync_dest=os.path.join(RSYNC_LIVE,os.getenv('GBSDBBUILD_DL_POSTFIX'))
        for subdir in self.obs_prj.split(':'):
            if subdir != self.obs_prj.split(':')[-1]:
                obs_liverepo=os.path.join(obs_liverepo,subdir+':')
                sync_dest=os.path.join(sync_dest,subdir+':')
            else:
                obs_liverepo=os.path.join(obs_liverepo,subdir)
                sync_dest=os.path.join(sync_dest,subdir)
            if not os.path.exists(obs_liverepo):
                os.mkdir(obs_liverepo)

        obs_liverepo=os.path.join(obs_liverepo,self.gbsbuild_tag.split('/')[-1])
        if not os.path.exists(obs_liverepo):
            os.mkdir(obs_liverepo)

        sync_dest=os.path.join(sync_dest,self.gbsbuild_tag.split('/')[-1])

        buildlogs_root = os.path.join(obs_liverepo,'buildlogs')
        os.mkdir(buildlogs_root)

        sync_out_dir=obs_liverepo

        profile_list=os.listdir(self.build_root+'/local/repos/')
        for profile in profile_list:
            build_result_repo_path=self.build_root+'/local/repos/'+profile+'/'
            live_repo_root=obs_liverepo+'/'+repository
            if not os.path.exists(live_repo_root):
                os.mkdir(live_repo_root)
            ###########copy arch.rpm files##############
            print '--------start copy arch.rpm files-----------------'
            for arch in rpm_arch_type_list.split(' '):
                if not os.path.exists(live_repo_root+'/'+arch):
                    os.mkdir(live_repo_root+'/'+arch)
                for gbs_build_arch in os.listdir(build_result_repo_path):
                    print 'copy source file dir: %s' %(build_result_repo_path+'/'+gbs_build_arch+'/RPMS/*.'+arch+'.rpm')
                    print 'copy destination dir: %s' %(live_repo_root+'/'+arch+'/')
                    for file_name in glob.glob(build_result_repo_path+'/'+gbs_build_arch+'/RPMS/*.'+arch+'.rpm'):
                        shutil.copy(file_name, live_repo_root+'/'+arch+'/')
                    #Handle for files whic are specified in _aggregate
                    print "Handle aggregate list: %s" %self.aggregate_list
                    for aggregate_rpm in self.aggregate_list:
                        for repository_source_target in aggregate_rpm['repository_source_target_list']:
                            source_repo = repository_source_target.split('##')[0]
                            target_repo = repository_source_target.split('##')[1]
                            if source_repo == repository:
                                if not os.path.exists(obs_liverepo+'/'+target_repo):
                                    os.mkdir(obs_liverepo+'/'+target_repo)
                                if not os.path.exists(obs_liverepo+'/'+target_repo+'/'+arch):
                                    os.mkdir(obs_liverepo+'/'+target_repo+'/'+arch)
                                for file_name in glob.glob(build_result_repo_path+'/'+gbs_build_arch+'/RPMS/'+aggregate_rpm['binary_rpm']+'*.'+arch+'.rpm'):
                                    shutil.copy(file_name, obs_liverepo+'/'+target_repo+'/'+arch+'/')
                #Remove folder if there is no file in arch directory
                if len(os.walk(live_repo_root+'/'+arch).next()[2]) == 0:
                    os.rmdir(live_repo_root+'/'+arch)
            ###############copy src.rpm files##############
            print '--------start copy src.rpm files-----------------'
            for gbs_build_arch in os.listdir(build_result_repo_path):
                if not os.path.exists(live_repo_root+'/src/'):
                    os.mkdir(live_repo_root+'/src/')
                for file_name in glob.glob(build_result_repo_path+'/'+gbs_build_arch+'/SRPMS/*.src.rpm'):
                    shutil.copy(file_name, live_repo_root+'/src/')
            ###############copy buildlog files##############
            print '--------start copy buildlog files-----------------'
            buildlogs_repository = os.path.join(buildlogs_root,repository)
            os.mkdir(buildlogs_repository)
            for gbs_build_arch in os.listdir(build_result_repo_path):
                os.mkdir(os.path.join(buildlogs_repository,gbs_build_arch))
                #index.html : overall gbs build status
                gbs_build_summary_src = os.path.join(build_result_repo_path,gbs_build_arch,SUMMARY_LOG)
                gbs_build_summary_dest = os.path.join(buildlogs_repository,gbs_build_arch,SUMMARY_LOG)
                print 'copy summary.log\nsrc file: %s, dest file: %s'\
                       %(gbs_build_summary_src,gbs_build_summary_dest)
                shutil.copy(gbs_build_summary_src,gbs_build_summary_dest)

                success_log_dest=os.path.join(buildlogs_repository,gbs_build_arch,'succeeded')
                fail_log_dest=os.path.join(buildlogs_repository,gbs_build_arch,'failed')
                os.mkdir(success_log_dest)
                os.mkdir(fail_log_dest)
                #succeeded packages
                success_log_root=os.path.join(build_result_repo_path,gbs_build_arch,'logs/success')
                fail_log_root=os.path.join(build_result_repo_path,gbs_build_arch,'logs/fail')
                for success_pkg in os.listdir(success_log_root):
                    src_file=success_log_root+'/'+success_pkg+'/log.txt'
                    spec_name=find_spec_name_from_build_log(src_file)
#                    print 'success_log_dest: %s ,success_pkg: %s' %(success_log_dest,success_pkg)
                    dest_file=success_log_dest+'/'+spec_name+'.buildlog.txt'
#                    dest_file=success_log_dest+'/'+re.sub('-\d+|.\d+','',success_pkg)+'.buildlog.txt'
                    print 'success_pkg: %s, spec_name: %s, dest_file: %s' %(success_pkg,spec_name,dest_file)
                    shutil.copy(src_file,dest_file)
                #failed packages
                for fail_pkg in os.listdir(fail_log_root):
                    src_file=fail_log_root+'/'+fail_pkg+'/log.txt'
                    spec_name=find_spec_name_from_build_log(src_file)
#                    dest_file=fail_log_dest+'/'+re.sub('-\d+|.\d+','',fail_pkg)+'.buildlog.txt'
                    dest_file=fail_log_dest+'/'+spec_name+'.buildlog.txt'
                    print 'fail_pkg: %s, spec_name: %s, dest_file: %s' %(fail_pkg,spec_name,dest_file)
                    shutil.copy(src_file,dest_file)
                #Remove folder if there is no file in arch directory
                if len(os.walk(success_log_dest).next()[2]) == 0:
                    os.rmdir(success_log_dest)
                if len(os.walk(fail_log_dest).next()[2]) == 0:
                    os.rmdir(fail_log_dest)

        #Finally, rsync live folder to download server
        print 'rsync sync_out_dir: %s, sync_dest: %s' %(sync_out_dir, sync_dest)
        sync(sync_out_dir, sync_dest)

        return sync_out_dir

    def get_rpm_info_from_gbs_build_log(self,live_out_dir,repo,arch):
        print '----start get_rpm_info_from_gbs_build_log-----------------------------------'

        rpm_file_info = []

        manifest_file = self.prj_src_root+'/'+repo+'/.repo/manifests/'+self.profile+'/'+repo+'/projects.xml'
        build_log_dir=os.path.join(live_out_dir,'buildlogs',repo,arch)
        build_rpm_root=os.path.join(live_out_dir,repo)

        with open(manifest_file,"rb") as f:
            manifest_content=f.read()

        for each_line in manifest_content.split('\n'):
            if each_line.find('<project name=') != -1:
                #processing for each git_repository
                #get spec file lists
                git_repository=each_line.split('<project name="')[1].split('"')[0]
                git_commit_id=each_line.split('revision="')[1].split('"')[0]
                git_repo_dir=os.path.join(self.prj_src_root,repo,git_repository)
                spec_files = []
                for each_file in os.listdir(os.path.join(git_repo_dir,'packaging')):
                    if each_file.endswith('.spec'):
                        spec_files.append(each_file.replace('.spec',''))
                print "git_repository: %s, spec file list: %s" %(git_repository,spec_files)

                #get rpm name
                for spec_file in spec_files:
                    spec_buildlog_name=spec_file+'.buildlog.txt'
#                    if buildlog_file in succeeded_build_log_list:
                    if os.path.exists(os.path.join(build_log_dir,'succeeded')):
                        for buildlog_file in os.listdir(os.path.join(build_log_dir,'succeeded')):
#                            if spec_buildlog_name == re.sub('-\d+|.\d+','',buildlog_file):
                            if spec_buildlog_name == buildlog_file:
                                build_status = 'S'
                                (build_time, built_rpm_files) = find_info_from_one_buildlog(os.path.join(build_log_dir,'succeeded',buildlog_file),spec_file)
                                for rpm_file_name in built_rpm_files:
                                    rpm_file_info.append({'git_repository':git_repository})
                                    rpm_file_info[-1]['git_commit_id']=git_commit_id
                                    rpm_file_info[-1]['spec_file_name']=spec_file
                                    rpm_file_info[-1]['rpm_file_name']=rpm_file_name
                                    rpm_file_info[-1]['build_time']=build_time
                                    rpm_file_info[-1]['status']=build_status
                                    rpm_file_info[-1]['status_reason']=''
                    if os.path.exists(os.path.join(build_log_dir,'failed')):
                        for buildlog_file in os.listdir(os.path.join(build_log_dir,'failed')):
#                            if spec_buildlog_name == re.sub('-\d+|.\d+','',buildlog_file):
                            if spec_buildlog_name == buildlog_file:
                                build_status = 'F'
                                status_reason = 'Build Failed'
                                rpm_file_info.append({'git_repository':git_repository})
                                rpm_file_info[-1]['git_commit_id']=git_commit_id
                                rpm_file_info[-1]['spec_file_name']=spec_file
                                rpm_file_info[-1]['rpm_file_name']=''
                                rpm_file_info[-1]['build_time']=''
                                rpm_file_info[-1]['status']=build_status
                                rpm_file_info[-1]['status_reason']=status_reason

        return rpm_file_info

def find_my_ip():
    try:
        return os.getenv('SSH_CONNECTION').split(' ')[2]
    except Exception as err:
        return '0.0.0.0'

def gbs_update_dashboard(raw_data, via='trigger', trigger_name=''):
    raw_data["_timestamp"] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
    raw_data["_worker"] = {"node_name": os.getenv('NODE_NAME'),
                           "build_job": os.getenv('JOB_NAME'),
                           "build_number": os.getenv('BUILD_ID'),
                           "ip": find_my_ip()}
    if via == 'trigger':
        if trigger_name:
            trigger_next("DASHBOARD_UPDATE_%s" % trigger_name, raw_data)
        else:
            trigger_next("DASHBOARD_UPDATE_%s" % raw_data.get("tag").replace('/', '_'), raw_data)
    else:
        try:
            encoded_data = base64.b64encode(json.dumps(raw_data))
            remote_jenkins_build_job(os.getenv('JENKINS_URL_INTERNAL'), \
                                     os.getenv('JENKINS_USER'), \
                                     os.getenv('JENKINS_PW'), \
                                     jobname=os.getenv('DASHBOARD_UPDATE_JOB_NAME', 'BUILD-MONITOR-GBS'), \
                                     data='TRIGGER_INFO="%s"' % (quote_plus(encoded_data)))
        except Exception as err:
            trigger_next("DASHBOARD_UPDATE_%s" % raw_data.get("tag").replace('/', '_'), raw_data)


