#!/usr/bin/env python
#
# Copyright (C) 2010, 2011, 2012, 2013, 2014 Intel, Inc.
#
#    This program is free software; you can redistribute it and/or
#    modify it under the terms of the GNU General Public License
#    as published by the Free Software Foundation; version 2
#    of the License.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
"""This script is used to create image"""

import re
import os
import sys
import shutil
import subprocess
import gzip
import xml.dom.minidom
import urllib2
import codecs
import datetime
import base64
import json

from xml.dom.minidom import parse,parseString
from collections import defaultdict
from StringIO import StringIO

from random import randint

from common.buildtrigger import trigger_info, trigger_next, remote_jenkins_build_job
from common.utils import sync, sync_get, set_permissions, Workdir
from urllib import quote_plus

def find_my_ip():
    try:
        return os.getenv('SSH_CONNECTION').split(' ')[2]
    except Exception as err:
        return '0.0.0.0'

def gbs_update_dashboard(raw_data, via='trigger', trigger_name=''):
    raw_data["_timestamp"] = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
    raw_data["_worker"] = {"node_name": os.getenv('NODE_NAME'),
                           "build_job": os.getenv('JOB_NAME'),
                           "build_number": os.getenv('BUILD_ID'),
                           "ip": find_my_ip()}
    if via == 'trigger':
        if trigger_name:
            trigger_next("DASHBOARD_UPDATE_%s" % trigger_name, raw_data)
        else:
            trigger_next("DASHBOARD_UPDATE_%s" % raw_data.get("tag").replace('/', '_'), raw_data)
    else:
        try:
            encoded_data = base64.b64encode(json.dumps(raw_data))
            remote_jenkins_build_job(os.getenv('JENKINS_URL_INTERNAL'), \
                                     os.getenv('JENKINS_USER'), \
                                     os.getenv('JENKINS_PW'), \
                                     jobname=os.getenv('DASHBOARD_UPDATE_JOB_NAME', 'BUILD-MONITOR-GBS'), \
                                     data='TRIGGER_INFO="%s"' % (quote_plus(encoded_data)))
        except Exception as err:
            trigger_next("DASHBOARD_UPDATE_%s" % raw_data.get("tag").replace('/', '_'), raw_data)

# Returns the list of baseurls from kickstart configuration.
def get_base_url(kickstart,buildid):
    baseurl = []
    for line in kickstart.split('\n'):
        if line.startswith('repo') and line.find('--baseurl') > 0:
           burl = re.search("(?P<url>https?://[^\s]+)", line).group('url')
           if burl.find('@BUILD_ID@') >0 :
               burl =  burl.replace('@BUILD_ID@',buildid)
           baseurl.append(burl)
    return baseurl


# Returns the url location of filelist.xml.gz
def get_gz_file_url(baseurl):
    repomdurl = os.path.join(baseurl, 'repodata/repomd.xml')
    response = urllib2.urlopen(repomdurl)
    xml_res = response.read()
    DOMTree = xml.dom.minidom.parseString(xml_res)
    collection = DOMTree.documentElement
    data = collection.getElementsByTagName("data")
    gz_url = None
    for d in data:
        if d.hasAttribute('type') and d.getAttribute('type') == 'filelists':
            gz_file_name = d.getElementsByTagName('location')[0].getAttribute('href')
            gz_url = repomdurl.replace('repodata/repomd.xml',gz_file_name)
            break
    return gz_url

#  Returns the decompressed xml data from given url location
def decompress_url_data(gzurl):
    if gzurl is None:
        return ''
    request = urllib2.Request(gzurl)
    request.add_header('Accept-encoding', 'gzip')
    response = urllib2.urlopen(request)
    buf = StringIO( response.read())
    f = gzip.GzipFile(fileobj=buf)
    xml_data = f.read()
    f.close()
    return xml_data

# Generates .files with filelist of packages
def gen_pkg_filelist(fields,base_path):

    fpkg= os.path.join(base_path,fields['name'],fields['buildid']+'_'+fields['name']+'.packages')
    fout= os.path.join(base_path,fields['name'],fields['buildid']+'_'+fields['name']+'.files')

    if not os.path.isfile(fpkg):
        print 'The file %s doesn\'t exists. Can\'t generate .files'%(fpkg)
        return -1

    baseurl = get_base_url(fields['kickstart'],fields['buildid'])

    if not baseurl:
        print 'No baseurl found in kick start file...'
        return -1
    pkgdict = defaultdict()

    for turl in baseurl:
        print 'baseurl %s'%(turl)
        gzurl =  get_gz_file_url(turl)
        print 'filelists url %s'%(gzurl)
        xml_content =  decompress_url_data(gzurl)

        DOMTree = xml.dom.minidom.parseString(xml_content)
        collection = DOMTree.documentElement
        packages = collection.getElementsByTagName("package")

        # The contents of xml are converted to dictionary in which key format is <package_name>.<arch> <version>-<release> and value is list of files for the package
        for pkg in packages:
            name_arch=  pkg.getAttribute('name')+'.'+pkg.getAttribute('arch')
            f = pkg.getElementsByTagName('version') [0]
            ver_rel= f.getAttribute('ver') + '-'+f.getAttribute('rel')
            pkgdict[name_arch+' '+ver_rel] = [ f.childNodes[0].data  for f in   pkg.getElementsByTagName('file') ]



    print 'Writing the list of files in packages to '+fout
    # For each package in the .packages file, get the list of pkg files from dictionary and writes them to .files
    with open (fpkg,'r') as pkg_f, codecs.open (fout,'w','utf-8') as out_f:
        for line in pkg_f:
            key =  line.split()[0]+' '+line.split()[1]
            out_f.write('\n\n'+key)
            if key in pkgdict.keys():
                for t in pkgdict[key]:
                    out_f.write('\n\t'+t)


def get_xml(image_dir, ks_name):
    """ Get the xml the image, return the body as string """

    for root, _subdir, files in os.walk(image_dir):
        for fname in files:
            if fname.endswith("%s.xml" % (ks_name)):
                return open(os.path.join(root,
                                         fname)).read()

    return ''


def convert_image_dir(image_dir, repo_name, image_name):
    """
    Insert repo name into image directory hierarchy.
    Origin image directory hierarchy: images/{image_name}
    The new one is: images/{repo_name}/{image_name}
    """

    with Workdir(image_dir):
        os.makedirs(repo_name)
        if not os.path.exists(image_name):
            return
        shutil.move(image_name, repo_name)


def run_inside_vm(vm_image, vm_memory, vm_cpus, basedir):
    """
    Run mic inside VM
    basedir(usually $WORKSPACE/mic) is shared by VM as Plan9 mount
    for cache and results
    """
    # Create a unique MAC address by EXECUTOR_NUMBER,
    # SSH_CONNECTION, BUILD_NUMBER, PID
    ssh_connection = os.getenv('SSH_CONNECTION')
    try:
        lastip = int(ssh_connection.split(' ')[2].split('.')[3])
    except IndexError:
        # Use random if ssh_connection has a wrong format or doesn't exist.
        print 'ssh_connection is %s, it is a incorrect format ,'\
              'random instead' % ssh_connection
        lastip = randint(0x00, 0xff)
    exeid = int(os.getenv('EXECUTOR_NUMBER', '1')) % 256
    processid = os.getpid() % 256
    buildid = int(os.getenv('BUILD_NUMBER',
                            randint(0x00, 0xff))) % 256
    # lastip from SSH_CONNECTION
    # exeid from EXECUTOR_NUMBER
    # processid from pid
    # buildid from BUILD_NUMBER
    mac_address = '52:%02x:%02x:%02x:%02x:%02x' % \
                  (lastip, exeid, processid,
                   buildid, randint(0x00, 0xff))
    cmd = 'qemu-system-x86_64 -machine accel=kvm:xen:tcg -name '\
          'opensuse -M pc -m %d -smp %d -vga none -drive file=%s,'\
          'snapshot=on -nographic -virtfs local,id=test_dev,'\
          'path=%s,security_model=mapped,mount_tag=share '\
          '-net nic,macaddr=%s -net user' % \
          (vm_memory, vm_cpus, vm_image, basedir, mac_address)

    subprocess.call(cmd, stdout=sys.stdout,
                    stderr=sys.stderr, shell=True)
    # read mic status from file
    try:
        with open(os.path.join(basedir, 'status')) as statusf:
            status = statusf.read().strip()
        if status == 'success':
            return 0
        else:
            return -1
    except IOError:
        return -1


def main():
    """The main body"""
    buildmonitor_enabled = os.getenv("BUILDMONITOR_ENABLED", "0") != "0"
    print 'buildmonitor_enabled(%s)\n' % (buildmonitor_enabled)
    if buildmonitor_enabled:
        bm_start_datetime = datetime.datetime.now()
    # Check if environment variables are set
    for var in ('WORKSPACE', 'IMG_SYNC_DEST_BASE'):
        if not os.getenv(var):
            print 'Error: environment variable %s is not set' % var
            return 1

    fields = trigger_info(os.getenv('TRIGGER_INFO'))

    # Don't need build monitor for gbsfullbuild
    if fields['repo_path'].find(os.getenv("GBSFULLBUILD_DL_POSTFIX")) != -1:
        buildmonitor_enabled = 0

    # Update dashboard gbs build status
    if buildmonitor_enabled:
        mode = "image_started"
        gbs_update_data = {"tag" : fields['gbsbuild_tag'],
                           "mode" : mode,
                           "reason" : {"ks_name":fields['name'], "repo":fields['repo']}}
        gbs_update_dashboard(gbs_update_data,via='trigger',trigger_name=mode)

    # Check if we've got required fields in TRIGGER_INFO
    for field in ('kickstart', 'name', 'buildid', 'repo_path'):
        if field not in fields:
            print 'Error: TRIGGER_INFO doesn\'t contain %s'
            return 1

    build_id = fields["buildid"]
    name = fields['name']

    # Prepare working directory
    basedir = os.path.join(os.getenv('WORKSPACE'), 'mic')
    if os.path.exists(basedir):
        shutil.rmtree(basedir)
    outdir = os.path.join(basedir, 'out')
    os.makedirs(outdir)

    url = os.path.join(fields.get('url_pub_base', ''),
                       fields['repo_path'], fields['images_path'])

    ksf = os.path.join(outdir, '%s.ks' % name)
    with open(ksf, 'w') as ks_fh:
        ks_fh.write(fields["kickstart"])
    if int(os.getenv('USE_VM')):
        vm_image = os.getenv("VM_IMAGE",
                             os.path.join(os.getenv('JENKINS_HOME'),
                                          'mic-appliance'))
        # check if tarball exists
        if not vm_image or not os.access(vm_image, os.R_OK):
            print 'VM image %s is not found' % vm_image
            return -1

        # Add --release option to mic command line
        with open(os.path.join(outdir, 'command'), 'w') as fcmdl:
            fcmdl.write('--release %s' % build_id)

        print 'starting mic inside VM to create image'
        sys.stdout.flush()
        ret = run_inside_vm(vm_image, int(os.getenv("VM_MEMORY", 8192)),
                            int(os.getenv("VM_CPUS", 8)), basedir)
        # workaround for qemu/9p bug in mapping permissions
        set_permissions(outdir, (0o644, 0o755))
    else:
        log = os.path.join(outdir, '%s_%s.log' % (build_id, name))
        cache = os.path.join(basedir, 'cache')
        if not os.path.exists(cache):
            os.makedirs(cache)

        # Use separate mic.conf file for each thread.
        if os.getenv('MIC_WORK_DIR') and os.getenv('MIC_CONF_FILE'):
            mic_conf_file = os.getenv('MIC_CONF_FILE')
            print 'Set mic conf file: %s' % mic_conf_file
            mic_cmd = "sudo /usr/bin/mic cr auto -c %s %s --release %s -o %s -k %s "\
                      '--logfile=%s' % (mic_conf_file, ksf, build_id, outdir, cache, log)
        else:
            mic_cmd = "sudo /usr/bin/mic cr auto %s --release %s -o %s -k %s "\
                      '--logfile=%s' % (ksf, build_id, outdir, cache, log)

        sys.stdout.flush()
        print 'starting mic to create image: %s' % mic_cmd
        ret = subprocess.call(mic_cmd,
                              stdout=sys.stdout,
                              stderr=sys.stderr,
                              shell=True)

        os.system('sudo chmod 0777 -R %s' % basedir)

    status = 'success'
    if ret:
        print 'Error: mic returned %d' % ret
        status = 'failed'
    # sync image, logs to SYNC_DEST
    sync_dest = os.path.join(os.getenv('IMG_SYNC_DEST_BASE'),
                             fields['repo_path'])

    sync_src = os.path.join(outdir, build_id)

    # Generates .files which contains package name and its file list
    #gen_pkg_filelist(fields,os.path.join(sync_src,'images'))

    convert_image_dir(os.path.join(sync_src, 'images'),
                      fields['repo'], fields['name'])

    # Copy logfile to sync directory
    if not int(os.getenv('USE_VM')):
        try:
            target_log = os.path.join(outdir, build_id, 'images', \
                                      fields['repo'], name, os.path.basename(log))
            if not os.path.exists(os.path.dirname(target_log)):
                os.makedirs(os.path.dirname(target_log))
            shutil.copy(log, target_log)
        except Exception as err:
            print 'LOGCPERROR: %s' % err

    sync_status = 'success'

    for loop in range(2):
        #TODO: If download_num differs from note, do not proceed!
        if 'download_num' in fields:
            try:
                if 0 == sync_get(os.path.join(sync_dest, 'buildinfo.in'), 'tmp_download_num'):
                    H = dict(line.strip().split('=') for line in open('tmp_download_num'))
                    print H
                    if int(H['download_num']) != int(fields['download_num']):
                        print 'DO NOT sync! download_num differs: %s, %s' \
                              % (fields['download_num'], H['download_num'])
                        return
            except Exception, err:
                print 'Error while reading note! %s' % str(err)
        if sync(sync_src, sync_dest):
            print "Retry sync %s to %s" % (sync_src, sync_dest)
            sync_status = 'failed'
        else:
            print "Success sync %s to %s" % (sync_src, sync_dest)
            sync_status = 'success'
            break

    if(sync_status == 'failed'):
        print "Error: unable to sync %s to %s" % (sync_src, sync_dest)
        status = 'failed'

    # Update dashboard gbs build status
    if buildmonitor_enabled:
        gbs_img_status = 'S'
        if status == 'failed':
            gbs_img_status = 'F'
        mode = "image_finished"
        gbs_update_data = {"tag" : fields['gbsbuild_tag'],
                           "mode" : mode,
                           "reason" : {"ks_name":fields['name'],"status":gbs_img_status}}
        gbs_update_dashboard(gbs_update_data,via='trigger',trigger_name=mode)


    xml_string = get_xml(os.path.join(outdir, build_id), name)

    # All project will trigger the post-image-creation
    # Trigger info for post image creation job
    data = {"image_xml": xml_string,
            "name": fields['name'],
            "project": fields.get('project', ''),
            "status": status,
            "url": url,
            "build_id": build_id,
            "repo": fields['repo'],
            }
    if 'download_num' in fields:
        data['download_num'] =  int(fields['download_num'])

    trigger_next("POST-IMAGE-CREATION", data)
    if status == 'success':
        print "The build was successful."
        fields["image_xml"] = xml_string
        fields["status"] = status
        fields["url"] = url
        trigger_next("IMAGE-TESTING", fields)

        if buildmonitor_enabled:
            bm_img_url = os.path.join(sync_src, fields['images_path'])

            # get bm_img_name from 'manifest.json' file
            search_name = 'image_files'
            manifest_name = 'manifest.json'
            bm_manifest_path = os.path.join(bm_img_url, manifest_name)
            with open(bm_manifest_path,'r') as f:
                bm_dict = eval(f.read())
                for key, val in bm_dict.iteritems():
                    if isinstance(val, dict) and val.has_key(search_name):
                        bm_img_name = val.get(search_name)[0]

            bm_img_path = os.path.join(bm_img_url, bm_img_name)
            print '[%s] bm_img_path(%s))\n' % (__file__, bm_img_path)

            if os.path.isfile(bm_img_path):
                bm_img_size = os.path.getsize(bm_img_path)
                #print '[%s] bm_img_path(%s), bm_img_size(%s)\n' \
                #      % (__file__, bm_img_path, bm_img_size)
            else:
                print '[%s] %s does not exist!!\n' % (__file__, bm_img_path)
                bm_img_size = 0
            bm_end_datetime = datetime.datetime.now()
            bm_stage = 'GBSDBBuild_Image'
            bm_data = {"bm_stage": bm_stage,
                       "status" : status,
                       "fields" : fields,
                       "bm_start_datetime" : str(bm_start_datetime),
                       "bm_end_datetime" : str(bm_end_datetime),
                       "build_id": build_id,
                       "bm_img_size" : bm_img_size
                      }
            trigger_next("BUILD-MONITOR", bm_data)
    else:
        if buildmonitor_enabled:
            fields["image_xml"] = xml_string
            fields["status"] = status
            fields["url"] = url
            bm_end_datetime = datetime.datetime.now()
            bm_stage = 'GBSDBBuild_Image'
            bm_data = {"bm_stage": bm_stage,
                       "status" : status,
                       "fields" : fields,
                       "bm_start_datetime": str(bm_start_datetime),
                       "bm_end_datetime" : str(bm_end_datetime),
                       "build_id": build_id
                      }
            trigger_next("BUILD-MONITOR", bm_data)
        return -1

if __name__ == "__main__":
    sys.exit(main())

