#!/bin/sh
#
# OpenVAS
# $Id: openvas-scapdata-sync.in 18082 2013-10-18 07:44:29Z timopollmeier $
# Description: Synchronize with SCAP data feed.
#
# Authors:
# Henri Doreau <henri.doreau@greenbone.net>
# Timo Pollmeier <timo.pollmeier@greenbone.net>
#
# Copyright:
# Copyright (C) 2011-2013 Greenbone Networks GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2,
# or, at your option, any later version as published by the Free
# Software Foundation
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

# configure SCAP_DIR where we will sync SCAP data
if [ -z "$SCAP_DIR" ]; then
  OPENVASSD=`which openvassd`
  if [ -z "$OPENVASSD" ] ; then
    echo "[e] Error: openvassd is not in the path, could not determine SCAP directory."
    exit 1
  else
    # get the parent directory of the plugins
    SCAP_DIR=`openvassd -s | awk -F" = " '/^plugins_folder/ { print $2 }' | sed -s 's/\(^.*\)\/plugins/\1/'`
    # suffix it with "scap-data" which is our target (destination) directory
    SCAP_DIR="$SCAP_DIR/scap-data"
  fi
fi
SEC_DB="$SCAP_DIR/scap.db"

# private directory
if [ -z "$PRIVATE_SUBDIR" ]
then
  PRIVATE_SUBDIR="private"
fi

# delete options for rsync
RSYNC_DELETE="--delete --exclude \"/scap.db\" --exclude \"$PRIVATE_SUBDIR/\""

# Script and feed information which will be made available to user through
# command line options and automated tools.
SCRIPT_NAME="openvas-scapdata-sync"
VERSION=4.0.4
SCAP_RES_DIR=/usr/share/openvas/scap
RESTRICTED=0

TIMESTAMP="$SCAP_DIR/timestamp"

if [ -z "$FEED_NAME" ] ; then
  FEED_NAME="OpenVAS SCAP Feed"
fi

if [ -z "$FEED_VENDOR" ] ; then
  FEED_VENDOR="The OpenVAS Project"
fi

if [ -z "$FEED_HOME" ] ; then
  FEED_HOME="http://www.openvas.org/"
fi

# The URL of the plugin feed
if [ -z "$OV_RSYNC_FEED" ]; then
  OV_RSYNC_FEED=rsync://feed.openvas.org:/scap-data
  # An alternative syntax which might work if the above doesn't:
  # OV_RSYNC_FEED=rsync@feed.openvas.org::scap-data
fi

if [ -z "$OV_HTTP_FEED" ]; then
  OV_HTTP_FEED=http://www.openvas.org/openvas-scap-data-current.tar.bz2
fi

if [ -z "$TMPDIR" ]; then
  SYNC_TMP_DIR=/tmp
  # If we have mktemp, create a temporary dir (safer)
  if [ -n "`which mktemp`" ]; then
    SYNC_TMP_DIR=`mktemp -t -d openvas-scap-data-sync.XXXXXXXXXX` || { echo "ERROR: Cannot create temporary directory for file download" >&2; exit 1 ; }
    trap "rm -rf $SYNC_TMP_DIR" EXIT HUP INT TRAP TERM
  fi
else
  SYNC_TMP_DIR="$TMPDIR"
fi

# When set to 1, all OVAL definitions will be updated, not just new ones.
# This is usually done by database migrations.
REBUILD_OVAL=0
# Whether to output extra data
VERBOSE=0

do_help () {
  echo "$0: Sync SCAP data using different protocols"
  echo " --rsync		sync with rsync (default)"
  echo " --refresh		update database without downloading feed data"
  echo " --refresh-private	update database only using private data"
  echo " --check		just checksum check"
  echo "OpenVAS administrator functions:"
  echo " --selftest	perform self-test"
  echo " --identify	display information"
  echo " --version	display version"
  echo " --describe	display current scap feed info"
  echo " --feedversion   display current scap feed version"
  echo " --dst-dir <dir>	SCAP destination directory"
  echo "Options:"
  echo " --verbose       enable verbose log messages"
  echo ""
  echo "Environment variables:"
  echo "SCAP_DIR		where to extract plugins (absolute path)"
  echo "OV_RSYNC_FEED	URL of rsync feed"
  echo "OV_HTTP_FEED	URL of http feed"
  echo "TMPDIR		temporary directory used to download the files"
  echo "PRIVATE_SUBDIR  subdirectory to exclude from deletion by rsync"
  echo "Note that you can use standard ones as well (e.g. http_proxy) for wget/curl"
  echo ""
  exit 0
}

CMD_RSYNC=`which rsync`
CMD_SQLITE=`which sqlite3`
SQLITE="sqlite3 -noheader"
TMP_SCAP="$SYNC_TMP_DIR/openvas-feed-`date +%F`-$$.tar.bz2"

chk_system_tools () {
  echo "[i] Searching for required system tools (look for warnings/errors)..."

  if [ -z "$CMD_RSYNC" ]; then
    echo "[w] Warning: RSYNC not found";
  fi

  if [ -z "$CMD_SQLITE" ]; then
    echo "[e] Error: sqlite3 not found (required)";
    exit 1
  fi

#  if [ -z "$CMD_RSYNC" -a -z "$CMD_WGET" -a -z "$CMD_CURL" ]; then
  if [ -z "$CMD_RSYNC" ]; then
    SELFTEST_FAIL=1
  fi

  echo "[i] If you did not get any warnings, that means you have all tools required"
}

do_rsync () {
  if [ -z "$CMD_RSYNC" ]; then
    echo "[w] rsync not found!"
  else
    echo "[i] Using rsync: $CMD_RSYNC"
    echo "[i] Configured SCAP data rsync feed: $OV_RSYNC_FEED"
    mkdir -p "$SCAP_DIR"
    eval "$CMD_RSYNC -ltvrP $RSYNC_DELETE \"$OV_RSYNC_FEED\" \"$SCAP_DIR\""
    if [ $? -ne 0 ] ; then
      echo "Error: rsync failed. Your SCAP data might be broken now."
      exit 1
    fi
  fi
}

do_wget () {
  echo "[w] Download of SCAP data via HTTP is currently not supported!"
  exit 1
}

do_curl () {
  echo "[w] Download of SCAP data via HTTP is currently not supported!"
  exit 1
}

do_self_test () {
  chk_system_tools
}

do_describe () {
  echo "This script synchronizes a SCAP collection with the '$FEED_NAME'."
  echo "The '$FEED_NAME' is provided by '$FEED_VENDOR'."
  echo "Online information about this feed: '$FEED_HOME'."

}

do_feedversion () {
  if [ -r $TIMESTAMP ] ; then
      echo `cat $TIMESTAMP`
  fi
}

show_intro () {
  echo "[i] This script synchronizes a SCAP data directory with the OpenVAS one."
  echo "[i] SCAP dir: $SCAP_DIR"
}

do_sync () {
  if [ -z "$CMD_RSYNC" ] ; then
    echo -n "[e] no utility available in PATH environment variable to download plugins"
    exit 1
  else
    echo "[i] Will use rsync"
    do_rsync
  fi
}

reinit () {
  echo "[i] Major change in internal SCAP data structures."
  echo "[i] Reinitialization of database necessary."
  echo "[i] This update might take a while.."
  $SQLITE $SEC_DB < $SCAP_RES_DIR/scap_db_init.sql
}

db_migrate_9 () {
  echo "[i] Migrating database to version 9."
  sqlite3 "$SEC_DB" "DROP INDEX IF EXISTS afp_idx;
                     CREATE INDEX afp_cpe_idx ON affected_products (cpe);
                     CREATE INDEX afp_cve_idx ON affected_products (cve);
                     UPDATE meta SET value = '9' WHERE name = 'database_version';"
  echo "[i] Migration done."
}

db_migrate_10 () {
  echo "[i] Migrating database to version 10."
  sqlite3 "$SEC_DB"  "DROP TRIGGER affected_delete;
                      CREATE TRIGGER affected_delete AFTER DELETE ON affected_products
                      BEGIN
                        UPDATE cpes set max_cvss = 0.0 WHERE id = old.cpe;
                        UPDATE cpes set cve_refs = cve_refs -1 WHERE id = old.cpe;
                      END;
                      UPDATE meta
                        SET value = '10'
                        WHERE name = 'database_version';"
  echo "[i] Migration done."
}

db_migrate_11 () {
  echo "[i] Migrating database to version 11."
  if [ 1 != "$REFRESH_PRIVATE_ONLY" ]
  then
    sqlite3 "$SEC_DB"  "ALTER TABLE ovaldefs
                          ADD COLUMN status TEXT;
                        UPDATE meta
                          SET value = '11'
                          WHERE name = 'database_version';
                      "
    REBUILD_OVAL=1
    echo "[i] Migration done."
  else
    echo "[w] Migration to version 11 requires refresh of all SCAP data. Aborting."
    exit 1
  fi
}

check_db_version () {
  DB_VERSION=`$SQLITE $SEC_DB "select value from meta where name = 'database_version';" 2>/dev/null || echo 0`
  case "$DB_VERSION" in
    0) reinit;;
    1) reinit;;
    2) reinit;;
    3) reinit;;
    4) reinit;;
    5) reinit;;
    6) reinit;;
    7) reinit;;
    8) db_migrate_9
       db_migrate_10
       db_migrate_11;;
    9) db_migrate_10
       db_migrate_11;;
   10) db_migrate_11;;
  esac
}

init_sec_db_update () {
  if [ ! -f $SEC_DB ]
  then
    echo "[i] Initializing scap database"
    $SQLITE $SEC_DB < $SCAP_RES_DIR/scap_db_init.sql
    DB_LASTUPDATE=0
    CVE_REFDATE=0
    OVAL_REFDATE=0
  else
    check_db_version
    CVE_REFDATE=`$SQLITE $SEC_DB "SELECT date(max(modification_time),'unixepoch') from cves;" | tr -d "-"`
  fi

  DB_LASTUPDATE=`$SQLITE $SEC_DB "select value from meta where name = 'last_update';"`

  if [ -z "$CVE_REFDATE" ]
  then
    CVE_REFDATE=0
  fi

  if [ -z "$DB_LASTUPDATE" ]
  then
    # Happens when initial sync was aborted
    echo "Error: Inconsistent data. Resetting SCAP database."
    rm -f $SEC_DB
    $SQLITE $SEC_DB < $SCAP_RES_DIR/scap_db_init.sql
    CVE_REFDATE=0
    OVAL_REFDATE=0
    DB_LASTUPDATE=0
  fi
}

list_oval_files_sorted () {
for line in `list_oval_files_sortable "$1" | sort`
  do
    echo ${line#????????}
  done
}

list_oval_files_sortable () {
for ovalfile in `find "$1" -name "*.xml" -type f`
  do
    timestamp_raw=""
    timestamp_raw=`xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" 2>/dev/null`

    if [ -n "$timestamp_raw" ] \
       && [ -n `echo "$timestamp_raw" | egrep -x "[0-9]{4}\-(0[0-9]|1[012])\-([0-2][0-9]|3[01])[[:alnum:]]*"` ]
    then
      timestamp="`echo "$timestamp_raw" | sed -e 's/T.*//' | tr -d "-"`"
      echo "$timestamp$ovalfile"
    fi
  done
}

update_sec_db_private () {
  SCAP_PRIVATE_DIR="$SCAP_DIR/$PRIVATE_SUBDIR"
  if [ -d "$SCAP_PRIVATE_DIR" ]
  then
    echo "[i] Updating user defined data"

    echo "[i] Updating user OVAL definitions"
    xmlcount=0
    xmlcount=$(find "$SCAP_PRIVATE_DIR/oval/" -name "*.xml" -type f 2> /dev/null | wc -l)
    if [ -n "$xmlcount" ] && [ $xmlcount -ne 0 ]
    then
      if [ -z "$oval_files_sorted" ]
      then
        oval_files_sorted=`list_oval_files_sorted "$SCAP_DIR/oval/"`
        ovaldir_xml_files=`find "$SCAP_DIR/oval/" -name "*.xml" -type f 2> /dev/null`

        for xml_file in $ovaldir_xml_files
        do
          if [ -z `echo "$oval_files_sorted" | grep -x "$xml_file"` ]
          then
            echo "[w] Cannot find valid OVAL timestamp in '$xml_file'."
          fi
        done
      fi

      oval_files_sorted_private=`list_oval_files_sorted "$SCAP_PRIVATE_DIR/oval/"`
      ovaldir_xml_files=`find "$SCAP_PRIVATE_DIR/oval/" -name "*.xml" -type f 2> /dev/null`

      for xml_file in $ovaldir_xml_files
      do
        if [ -z `echo "$oval_files_sorted_private" | grep -x "$xml_file"` ]
        then
          echo "[w] Cannot find valid OVAL timestamp in '$xml_file'."
        fi
      done

      for non_xml_file in `find "$SCAP_PRIVATE_DIR/oval/" -type f -and -not -name "*.xml" 2> /dev/null`
      do
        if [ "${non_xml_file%%.asc}" = "${non_xml_file}" ] || ! [ -f "${non_xml_file%%.asc}" ]
        then
          echo "[w] Found non-XML and non-signature file '$non_xml_file'."
        fi
      done

      if [ -n "$oval_files_sorted" ]
      then
        for ovalfile in $oval_files_sorted_private
        do
          if [ `stat -c "%Y" "$ovalfile" | cut -d " " -f 1 | tr -d "-"` -ge $DB_LASTUPDATE ] || [ 1 = "$REBUILD_OVAL" ]
          then
            oval_timestamp=`xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" | sed -e 's/T.*//' | tr -d "-"`

            if [ 1 = "$REBUILD_OVAL" ]
            then
              OVAL_REFDATE=0
            else
              OVAL_REFDATE=`$SQLITE $SEC_DB "SELECT date(max(modification_time),'unixepoch') FROM ovaldefs WHERE xml_file = '${ovalfile##$SCAP_DIR/}';" | tr -d "-"`
              if [ -z "$OVAL_REFDATE" ]
              then
                OVAL_REFDATE=0
              fi
            fi

            if [ $oval_timestamp -ge $OVAL_REFDATE ]
            then
              validation_message=`xsltproc "$SCAP_RES_DIR/oval_verify.xsl" "$ovalfile" 2>/dev/null`
              validation_return="$?"
              if [ 0 -ne $validation_return ] && [ -n "$validation_message" ]
              then
                if [ 10 -eq $validation_return ]
                then
                  echo "[w] Validation of file '$ovalfile' failed: $validation_message"
                else
                  echo "[w] Error parsing '$ovalfile'."
                fi
              else
                if ! [ "$validation_message" = "file valid" ]
                then
                  echo "[w] Validation of file '$ovalfile' failed: $validation_message"
                else
                  for feedfile in $oval_files_sorted
                  do
                    compare=`xsltproc "$SCAP_RES_DIR/ovaldef_list_ids.xsl" "$ovalfile"`
                    duplicates=`xsltproc "$SCAP_RES_DIR/ovaldef_list_ids.xsl" "$feedfile" | grep -Fx "$compare"`
                    duplicates_count=`echo "$duplicates" | wc -l`
                    if [ -n "$duplicates" ]
                    then
                      echo "[w] Skipping '$ovalfile', found $duplicates_count definition(s) that are duplicates of one(s) in '$feedfile'"
                      if [ 0 -ne $VERBOSE ]
                      then
                        for duplicate in $duplicates
                        do
                          echo "[w] Duplicate ID: $duplicate"
                        done
                      fi
                    fi
                  done
                  if [ 0 -eq $duplicates_count ]
                  then
                    echo "[i] Updating $ovalfile"
                    xsltproc --stringparam filename "${ovalfile##$SCAP_DIR/}" --stringparam refdate "$OVAL_REFDATE" "$SCAP_RES_DIR/oval_update.xsl" "$ovalfile" | \
                    $SQLITE $SEC_DB
                  fi
                fi
              fi
            else
              echo "[i] Skipping $ovalfile, file has older timestamp than latest OVAL definition in database."
            fi
          else
            echo "[i] Skipping $ovalfile, file is older than last revision."
          fi
        done
      fi
    else
      echo "[i] No user defined OVAL files found"
    fi

    echo "[i] Cleaning up user OVAL data"
    DIR_STR_LENGTH=$((`echo "$SCAP_DIR" | wc -c` + 1))

    oval_files_shortened=""
    if [ 0 != "$xmlcount" ]
    then
      for ovalfile in $oval_files_sorted_private
      do
        oval_files_shortened=$oval_files_shortened"', '"`echo $ovalfile | cut -c ${DIR_STR_LENGTH}- `
      done
      oval_files_shortened=${oval_files_shortened##"', "}"'"
    fi

    deleted=`$SQLITE "$SEC_DB" "SELECT DISTINCT xml_file FROM ovaldefs WHERE (xml_file NOT LIKE 'oval/%') AND (xml_file NOT IN ($oval_files_shortened));"`
    if [ -n "$deleted" ]
    then
      echo "[i] Removing definitions formerly inserted from:"
      echo "$deleted"
      $SQLITE "$SEC_DB" "DELETE FROM ovaldefs WHERE (xml_file NOT LIKE 'oval/%') AND (xml_file NOT IN ($oval_files_shortened));"
    fi

    $SQLITE $SEC_DB "update meta set value ='`date +%s`' where name = 'last_update';"
  else
    echo "[i] No user data directory '$SCAP_PRIVATE_DIR' found."
  fi
}

update_sec_db () {
  init_sec_db_update

  CPEBASE="$SCAP_DIR/official-cpe-dictionary_v2.2.xml"
  if [ -e $CPEBASE ]
  then
    if [ `stat -c "%Y" $CPEBASE | cut -d " " -f 1 | tr -d "-"` -ge $DB_LASTUPDATE ]
    then
      echo "[i] Updating CPEs"
      sed 's/&/&amp;/g' $CPEBASE | \
        xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cpe_youngerthan.xsl - | \
        xsltproc $SCAP_RES_DIR/cpe_update.xsl - | \
        $SQLITE $SEC_DB
    else
      echo "[i] Skipping CPEs, file is older than last revision"
    fi
  else
    echo "[w] No CPE dictionary found in $SCAP_DIR"
  fi

  xmlcount=$(ls $SCAP_DIR/nvdcve-2.0-*.xml 2> /dev/null | wc -l)
  updated_cves=0
  if [ $xmlcount -ne 0 ]
  then
    for cvefile in `ls $SCAP_DIR/nvdcve-2.0-*.xml`
    do
      if [ `stat -c "%Y" $cvefile | cut -d " " -f 1 | tr -d "-"` -ge $DB_LASTUPDATE ]
      then
        echo "[i] Updating $cvefile"
        xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cve_youngerthan.xsl $cvefile | \
          xsltproc $SCAP_RES_DIR/cve_update.xsl - | \
          $SQLITE $SEC_DB
        updated_cves=1
      else
        echo "[i] Skipping $cvefile, file is older than last revision"
      fi
    done
  else 
    echo "[w] No CVEs found in $SCAP_DIR"
  fi

  if [ 0 -ne $updated_cves ]
  then
    echo "[i] Updating CVSS scores and CVE counts"
    sqlite3 "$SEC_DB"  "
    PRAGMA recursive_triggers = OFF;
    UPDATE cpes
      SET max_cvss =
          (
            SELECT coalesce(max(cvss), 0.0)
            FROM cves
            WHERE id IN
              (
                SELECT cve
                FROM affected_products
                WHERE cpe=cpes.id
              )
          ),
        cve_refs =
          (
            SELECT count(cve)
            FROM affected_products
            WHERE cpe=cpes.id
          );
    "
  else
    echo "[i] No CVEs updated, skipping CVSS and CVE recount."
  fi

  echo "[i] Updating OVAL data"
  xmlcount=$(find "$SCAP_DIR/oval/" -name "*.xml" -type f 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then

    oval_files_sorted=`list_oval_files_sorted "$SCAP_DIR/oval/"`
    ovaldir_xml_files=`find "$SCAP_DIR/oval/" -name "*.xml" -type f 2> /dev/null`

    for xml_file in $ovaldir_xml_files
    do
      if [ -z `echo "$oval_files_sorted" | grep -x "$xml_file"` ]
      then
        echo "[w] Cannot find valid OVAL timestamp in '$xml_file'."
      fi
    done

    for ovalfile in $oval_files_sorted
    do
      if [ `stat -c "%Y" "$ovalfile" | cut -d " " -f 1 | tr -d "-"` -ge $DB_LASTUPDATE ] || [ 1 = "$REBUILD_OVAL" ]
      then
        oval_timestamp=`xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" | sed -e 's/T.*//' | tr -d "-"`

        if [ 1 = "$REBUILD_OVAL" ]
        then
          OVAL_REFDATE=0
        else
          OVAL_REFDATE=`$SQLITE $SEC_DB "SELECT date(max(modification_time),'unixepoch') FROM ovaldefs WHERE xml_file = '${ovalfile##$SCAP_DIR/}';" | tr -d "-"`
          if [ -z "$OVAL_REFDATE" ]
          then
            OVAL_REFDATE=0
          fi
        fi

        if [ $oval_timestamp -ge $OVAL_REFDATE ]
        then
          echo "[i] Updating $ovalfile"
          xsltproc --stringparam filename "${ovalfile##$SCAP_DIR/}" --stringparam refdate "$OVAL_REFDATE" "$SCAP_RES_DIR/oval_update.xsl" "$ovalfile" | \
          $SQLITE $SEC_DB
        else
          echo "[i] Skipping $ovalfile, file has older timestamp than latest OVAL definition in database."
        fi
      else
        echo "[i] Skipping $ovalfile, file is older than last revision."
      fi
    done
  else
    echo "[w] No XML files found in $SCAP_DIR/oval/."
  fi

  update_sec_db_private

  $SQLITE $SEC_DB "update meta set value ='`date +%s`' where name = 'last_update';"
}

do_refresh () {
  echo "[i] Refreshing database without feed sync."
  update_sec_db
}

do_refresh_private () {
  REFRESH_PRIVATE_ONLY=1
  init_sec_db_update
  update_sec_db_private
}

if [ -n "$1" ]; then
  while test $# -gt 0; do
    case "$1" in
      --help)
        do_help
        exit 0
        ;;
      --rsync)
        do_rsync
        exit 0
        ;;
      --wget)
        do_wget
        exit 0
        ;;
      --curl)
        do_curl
        exit 0
        ;;
      --verbose)
        VERBOSE=1
        ;;
      --refresh)
        do_refresh
        exit 0
        ;;
      --refresh-private)
        do_refresh_private
        exit 0
        ;;
      --check)
        exit 0
        ;;
      --version)
        echo $VERSION
        exit 0
        ;;
      --identify)
        echo "SCAPSYNC|$SCRIPT_NAME|$VERSION|$FEED_NAME|$RESTRICTED|SCAPSYNC"
        exit 0
        ;;
      --selftest)
        SELFTEST_FAIL=0
        do_self_test
        exit $SELFTEST_FAIL
        ;;
      --describe)
        do_describe
        exit 0
        ;;
      --feedversion)
        do_feedversion
        exit 0
        ;;
      --nvt-dir)
        SCAP_DIR="$2"
        shift
        ;;
    esac
    shift
  done
fi

show_intro
do_sync
update_sec_db

exit 0
