openvas-manager 6.0.9-2 / usr / sbin / openvas-scapdata-sync

#!/bin/sh
#
# OpenVAS
# $Id: openvas-scapdata-sync.in 21677 2015-02-18 15:01:24Z timopollmeier $
# Description: Synchronize with SCAP data feed.
#
# Authors:
# Henri Doreau <henri.doreau@greenbone.net>
# Timo Pollmeier <timo.pollmeier@greenbone.net>
#
# Copyright:
# Copyright (C) 2011-2013 Greenbone Networks GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

POSTGRES=0
SCHEMA=""
RECURSIVE_TRIGGERS_OFF="PRAGMA recursive_triggers = OFF;"
if [ "SQLITE3" = "POSTGRESQL" ]; then
  POSTGRES=1
  SCHEMA="scap."
  RECURSIVE_TRIGGERS_OFF=""
fi

# configure SCAP_DIR where we will sync SCAP data
if [ -z "$SCAP_DIR" ]; then
  SCAP_DIR=/var/lib/openvas/scap-data
fi
SEC_DB="$SCAP_DIR/scap.db"

# private directory
if [ -z "$PRIVATE_SUBDIR" ]
then
  PRIVATE_SUBDIR="private"
fi

# delete options for rsync
RSYNC_DELETE="--delete --exclude \"/scap.db\" --exclude \"$PRIVATE_SUBDIR/\""

# Script and feed information which will be made available to user through
# command line options and automated tools.
SCRIPT_NAME="openvas-scapdata-sync"
VERSION=6.0.9
SCAP_RES_DIR=/usr/share/openvas/scap
RESTRICTED=0

# Maximum number of retries if database is locked
if [ -z "$MAX_SQL_RETRIES" ]; then
  MAX_SQL_RETRIES="1" # 0 : try only once
fi

# Delay between retries
if [ -z "$SQL_RETRY_DELAY" ]; then
  SQL_RETRY_DELAY="10m" # allowed unit suffixes: see sleep command
fi

TIMESTAMP="$SCAP_DIR/timestamp"

if [ -z "$FEED_NAME" ] ; then
  FEED_NAME="OpenVAS SCAP Feed"
fi

if [ -z "$FEED_VENDOR" ] ; then
  FEED_VENDOR="The OpenVAS Project"
fi

if [ -z "$FEED_HOME" ] ; then
  FEED_HOME="http://www.openvas.org/"
fi

# The URL of the plugin feed
if [ -z "$OV_RSYNC_FEED" ]; then
  OV_RSYNC_FEED=rsync://feed.openvas.org:/scap-data
  # An alternative syntax which might work if the above doesn't:
  # OV_RSYNC_FEED=rsync@feed.openvas.org::scap-data
fi

if [ -z "$OV_HTTP_FEED" ]; then
  OV_HTTP_FEED=http://www.openvas.org/openvas-scap-data-current.tar.bz2
fi

if [ -z "$TMPDIR" ]; then
  SYNC_TMP_DIR=/tmp
  # If we have mktemp, create a temporary dir (safer)
  if [ -n "`which mktemp`" ]; then
    SYNC_TMP_DIR=`mktemp -t -d openvas-scap-data-sync.XXXXXXXXXX` || { echo "[e] Error: Cannot create temporary directory for file download" >&2; exit 1 ; }
    trap "rm -rf $SYNC_TMP_DIR" EXIT HUP INT TRAP TERM
  fi
else
  SYNC_TMP_DIR="$TMPDIR"
fi

# When set to 1, all OVAL definitions will be updated, not just new ones.
# This is usually done by database migrations.
REBUILD_OVAL=0
# Whether to output extra data
VERBOSE=0

# Split CVE data files into parts of this size in kB. 0 = no splitting
SPLIT_PART_SIZE=0

do_help () {
  echo "$0: Sync SCAP data using different protocols"
  echo " --rsync		sync with rsync (default)"
  echo " --migrate		migrate database without downloading feed data"
  echo " --refresh		update database without downloading feed data"
  echo " --refresh-private	update database only using private data"
  echo " --check		just checksum check"
  echo "OpenVAS administrator functions:"
  echo " --selftest	perform self-test"
  echo " --identify	display information"
  echo " --version	display version"
  echo " --describe	display current scap feed info"
  echo " --feedversion   display current scap feed version"
  echo "Options:"
  echo " --verbose       enable verbose log messages"
  if [ $POSTGRES -eq 1 ];
  then
    echo " --database       database name"
  fi
  echo ""
  echo "Environment variables:"
  echo "SCAP_DIR		where to extract plugins (absolute path)"
  echo "OV_RSYNC_FEED	URL of rsync feed"
  echo "OV_HTTP_FEED	URL of http feed"
  echo "TMPDIR		temporary directory used to download the files"
  echo "PRIVATE_SUBDIR  subdirectory to exclude from deletion by rsync"
  echo "Note that you can use standard ones as well (e.g. http_proxy) for wget/curl"
  echo ""
  exit 0
}

CMD_RSYNC=`which rsync`
CMD_SQLITE=`which sqlite3`
SQLITE="sqlite3 -noheader -bail"
TMP_SCAP="$SYNC_TMP_DIR/openvas-feed-`date +%F`-$$.tar.bz2"

chk_system_tools () {
  echo "[i] Searching for required system tools (look for warnings/errors)..."

  if [ -z "$CMD_RSYNC" ]; then
    echo "[w] Warning: RSYNC not found";
  fi

  if [ -z "$CMD_SQLITE" ]; then
    echo "[e] Error: sqlite3 not found (required)";
    exit 1
  fi

#  if [ -z "$CMD_RSYNC" -a -z "$CMD_WGET" -a -z "$CMD_CURL" ]; then
  if [ -z "$CMD_RSYNC" ]; then
    SELFTEST_FAIL=1
  fi

  echo "[i] If you did not get any warnings, that means you have all tools required"
}

do_rsync () {
  if [ -z "$CMD_RSYNC" ]; then
    echo "[w] rsync not found!"
  else
    echo "[i] Using rsync: $CMD_RSYNC"
    echo "[i] Configured SCAP data rsync feed: $OV_RSYNC_FEED"
    mkdir -p "$SCAP_DIR"
    eval "$CMD_RSYNC -ltvrP $RSYNC_DELETE \"$OV_RSYNC_FEED\" \"$SCAP_DIR\""
    if [ $? -ne 0 ] ; then
      echo "[e] Error: rsync failed. Your SCAP data might be broken now."
      exit 1
    fi
  fi
}

do_wget () {
  echo "[w] Download of SCAP data via HTTP is currently not supported!"
  exit 1
}

do_curl () {
  echo "[w] Download of SCAP data via HTTP is currently not supported!"
  exit 1
}

do_self_test () {
  chk_system_tools
}

do_describe () {
  echo "This script synchronizes a SCAP collection with the '$FEED_NAME'."
  echo "The '$FEED_NAME' is provided by '$FEED_VENDOR'."
  echo "Online information about this feed: '$FEED_HOME'."

}

do_feedversion () {
  if [ -r $TIMESTAMP ] ; then
      echo `cat $TIMESTAMP`
  fi
}

show_intro () {
  echo "[i] This script synchronizes a SCAP data directory with the OpenVAS one."
  if [ $POSTGRES -eq 1 ] ; then
    echo "[i] This script is for the PostgreSQL backend."
  else
    echo "[i] This script is for the SQLite3 backend."
  fi
  echo "[i] SCAP dir: $SCAP_DIR"
}

do_sync () {
  if [ -z "$CMD_RSYNC" ] ; then
    echo -n "[e] no utility available in PATH environment variable to download plugins"
    exit 1
  else
    echo "[i] Will use rsync"
    do_rsync
  fi
}

set_interrupt_trap () {
  trap "handle_interrupt $1" 2
}

handle_interrupt () {
  echo "$1:X" >&3
}

reset_sql_tries () {
  try_sql=1
  sql_retries=0
}

test_exit_codes () {
  try_sql=0
  if [ -n "$exit_codes" ]
  then
    for item in $exit_codes
    do
      command=`echo "$item" | cut -d':' -f1`
      code=`echo "$item" | cut -d':' -f2`
      if [ "X" = "$code" ]
      then
        echo "[e] $1: Sync script was interrupted"
        exit 1
      elif [ "sqlite3" = "$command" ] \
           && ( [ "5" = "$code" ] || [ "1" = "$code" ] )
      then
        if [ "5" = "$code" ]
        then
          echo "[w] SCAP database is locked"
        else
          echo "[w] Could not access SCAP database, file may be locked."
        fi

        if [ "$MAX_SQL_RETRIES" -gt "$sql_retries" ]
        then
          sql_retries=$((sql_retries + 1))
          echo "[i] Will try to access database again later in $SQL_RETRY_DELAY. (Retry $sql_retries of $MAX_SQL_RETRIES)"
          try_sql=1
          sleep "$SQL_RETRY_DELAY"
        else
          echo "[e] $1: Gave up trying to access SCAP database."
          exit 1
        fi
      else
        echo "[e] $1: $command exited with code $code"
        exit 1
      fi
    done
  fi
}

test_sql_exit () {
  exit_code=$?
  try_sql=0
  if ( [ "5" = "$exit_code" ] || [ "1" = "$exit_code" ] )
  then
    if [ "5" = "$exit_code" ]
    then
      echo "[w] SCAP database is locked."
    else
      echo "[w] Could not access SCAP database, file may be locked."
    fi

    if [ "$MAX_SQL_RETRIES" -gt "$sql_retries" ]
    then
      sql_retries=$((sql_retries + 1))
      echo "[i] Will try to access database again later in $SQL_RETRY_DELAY. (Retry $sql_retries of $MAX_SQL_RETRIES)"
      try_sql=1
      sleep "$SQL_RETRY_DELAY"
    else
      echo "[e] $1: Gave up trying to access SCAP database."
      exit 1
    fi
  elif [ 0 -ne "$exit_code" ]
  then
    echo "[e] $1: sqlite3 exited with code $exit_code."
    exit 1
  fi
}

reinit () {
  echo "[i] Major change in internal SCAP data structures."
  echo "[i] Reinitialization of database necessary."
  echo "[i] This update might take a while.."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    sql < $SCAP_RES_DIR/scap_db_init.sql
    test_sql_exit "Could not reinitialize SCAP database"
  done
}

db_migrate_9 () {
  echo "[i] Migrating database to version 9."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE "$SEC_DB" "DROP INDEX IF EXISTS afp_idx;
                      CREATE INDEX afp_cpe_idx ON affected_products (cpe);
                      CREATE INDEX afp_cve_idx ON affected_products (cve);
                      UPDATE meta SET value = '9' WHERE name = 'database_version';"
    test_sql_exit "Could not migrate SCAP database"
    echo "[i] Migration done."
  done
}

db_migrate_10 () {
  echo "[i] Migrating database to version 10."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE "$SEC_DB"  "DROP TRIGGER affected_delete;
                        CREATE TRIGGER affected_delete AFTER DELETE ON affected_products
                        BEGIN
                          UPDATE cpes set max_cvss = 0.0 WHERE id = old.cpe;
                          UPDATE cpes set cve_refs = cve_refs -1 WHERE id = old.cpe;
                        END;
                        UPDATE meta
                          SET value = '10'
                          WHERE name = 'database_version';"
    test_sql_exit "Could not migrate SCAP database"
  done
  echo "[i] Migration done."
}

db_migrate_11 () {
  echo "[i] Migrating database to version 11."
  if [ 1 != "$REFRESH_PRIVATE_ONLY" ]
  then
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      $SQLITE "$SEC_DB"  "ALTER TABLE ovaldefs
                            ADD COLUMN status TEXT;
                          UPDATE meta
                            SET value = '11'
                            WHERE name = 'database_version';
                        "
      test_sql_exit "Could not migrate SCAP database"
    done
    REBUILD_OVAL=1
    echo "[i] Migration done."
  else
    echo "[w] Migration to version 11 requires refresh of all SCAP data. Aborting."
    exit 1
  fi
}

db_migrate_12 () {
  echo "[i] Migrating database to version 12."
  if [ 1 != "$REFRESH_PRIVATE_ONLY" ]
  then
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      $SQLITE "$SEC_DB"  "DROP TABLE ovaldefs;
                          DROP INDEX IF EXISTS ovaldefs_idx;
                          CREATE TABLE ovaldefs (
                            id INTEGER PRIMARY KEY AUTOINCREMENT,
                            uuid UNIQUE,
                            name,
                            comment,
                            creation_time DATE,
                            modification_time DATE,
                            version INTEGER,
                            deprecated BOOLEAN,
                            def_class TEXT,
                            title TEXT,
                            description TEXT,
                            xml_file TEXT,
                            status TEXT
                          );
                          CREATE INDEX ovaldefs_idx ON ovaldefs (name);
                          CREATE TABLE ovalfiles (
                            id INTEGER PRIMARY KEY AUTOINCREMENT,
                            xml_file TEXT UNIQUE
                          );
                          CREATE UNIQUE INDEX ovalfiles_idx ON ovalfiles (xml_file);
                          CREATE TRIGGER ovalfiles_delete AFTER DELETE ON ovalfiles
                          BEGIN
                            DELETE FROM ovaldefs WHERE ovaldefs.xml_file = old.xml_file;
                          END;
                          UPDATE meta
                            SET value = '12'
                            WHERE name = 'database_version';
                        "
      test_sql_exit "Could not migrate SCAP database"
    done
    REBUILD_OVAL=1
    echo "[i] Migration done."
  else
    echo "[i] Migration to version 12 requires refresh of all SCAP data. Aborting."
    exit 1
  fi
}

db_migrate_13 () {
  echo "[i] Migrating database to version 13."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE "$SEC_DB"  "ALTER TABLE cpes ADD COLUMN nvd_id;
                        UPDATE cpes SET nvd_id = uuid, uuid = name;
                        UPDATE meta
                          SET value = '13'
                          WHERE name = 'database_version';
                      "
    test_sql_exit "Could not migrate SCAP database"
  done
  echo "[i] Migration done."
}

db_migrate_14 () {
  echo "[i] Migrating database to version 14."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE "$SEC_DB"  "ALTER TABLE ovaldefs ADD COLUMN max_cvss FLOAT;
                        ALTER TABLE ovaldefs ADD COLUMN cve_refs INTEGER;

                        CREATE TABLE affected_ovaldefs (
                          cve INTEGER NOT NULL,
                          ovaldef INTEGER NOT NULL,
                          FOREIGN KEY(cve) REFERENCES cves(id),
                          FOREIGN KEY(ovaldef) REFERENCES ovaldefs(id)
                        );
                        CREATE INDEX aff_ovaldefs_def_idx
                          ON affected_ovaldefs (ovaldef);
                        CREATE INDEX aff_ovaldefs_cve_idx
                          ON affected_ovaldefs (cve);

                        DROP TRIGGER IF EXISTS cves_delete;
                        CREATE TRIGGER cves_delete AFTER DELETE ON cves
                        BEGIN
                          DELETE FROM affected_products where cve = old.id;
                          DELETE FROM affected_ovaldefs where cve = old.id;
                        END;

                        CREATE TRIGGER affected_ovaldefs_delete AFTER DELETE
                          ON affected_ovaldefs
                        BEGIN
                          UPDATE ovaldefs
                            SET max_cvss = 0.0
                            WHERE id = old.ovaldef;
                        END;

                        UPDATE meta
                          SET value = '14'
                          WHERE name = 'database_version';
                      "
    test_sql_exit "Could not migrate SCAP database"
  done
  echo "[i] Migration done."
  REBUILD_OVAL=1
}

db_migrate_15 () {
  echo "[i] Migrating database to version 15."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    $SQLITE "$SEC_DB"  "CREATE INDEX IF NOT EXISTS cpes_by_creation_time_idx
                          ON cpes (creation_time);
                        CREATE INDEX IF NOT EXISTS cpes_by_modification_time_idx
                          ON cpes (modification_time);
                        CREATE INDEX IF NOT EXISTS cpes_by_cvss
                          ON cpes (max_cvss);

                        CREATE INDEX IF NOT EXISTS cves_by_creation_time_idx
                          ON cves (creation_time);
                        CREATE INDEX IF NOT EXISTS cves_by_modification_time_idx
                          ON cves (modification_time);
                        CREATE INDEX IF NOT EXISTS cves_by_cvss
                          ON cves (cvss);

                        UPDATE meta
                          SET value = '15'
                          WHERE name = 'database_version';
                      "
    test_sql_exit "Could not migrate SCAP database"
  done
  echo "[i] Migration done."
}

PSQL=""
psql_setup () {
  PSQL="psql -v ON_ERROR_STOP=1 -q --pset pager=off --no-align -d $1 -t"
}

sql_pg () {
  if [ "$#" -gt 0 ]
  then
    $PSQL -c "$1"
  else
    $PSQL -f -
  fi
  exit_code=$?
  if [ 0 -ne "$exit_code" ]
  then
    echo "[e] psql exited with code $exit_code for sql: $1."
    exit 1
  fi
}

sql () {
  if [ $POSTGRES -eq 1 ]
  then
    if [ "$#" -gt 0 ]
    then
      sql_pg "$1"
    else
      sql_pg $*
    fi
  else
    if [ "$#" -gt 0 ]
    then
      $SQLITE $SEC_DB "$1"
    else
      $SQLITE $SEC_DB $*
    fi
  fi
}

check_db_version () {
  DB_VERSION=`sql "select value from ${SCHEMA}meta where name = 'database_version';" 2>/dev/null | tr -d '\n\r' || echo 0`
  case "$DB_VERSION" in
    0) reinit;;
    1) reinit;;
    2) reinit;;
    3) reinit;;
    4) reinit;;
    5) reinit;;
    6) reinit;;
    7) reinit;;
    8) if [ $POSTGRES -eq 1 ]
       then
         reinit
       else
         db_migrate_9
         db_migrate_10
         db_migrate_11
         db_migrate_12
         db_migrate_13
         db_migrate_14
         db_migrate_15
       fi;;
    9) if [ $POSTGRES -eq 1 ]
       then
         reinit
       else
         db_migrate_10
         db_migrate_11
         db_migrate_12
         db_migrate_13
         db_migrate_14
         db_migrate_15
       fi;;
   10) if [ $POSTGRES -eq 1 ]
       then
         reinit
       else
         db_migrate_11
         db_migrate_12
         db_migrate_13
         db_migrate_14
         db_migrate_15
       fi;;
   11) if [ $POSTGRES -eq 1 ]
       then
         reinit
       else
         db_migrate_12
         db_migrate_13
         db_migrate_14
         db_migrate_15
       fi;;
   12) if [ $POSTGRES -eq 1 ]
       then
         reinit
       else
         db_migrate_13
         db_migrate_14
         db_migrate_14
       fi;;
   13) if [ $POSTGRES -eq 1 ]
       then
         reinit
       else
         db_migrate_14
         db_migrate_15
       fi;;
   14) if [ $POSTGRES -eq 1 ]
       then
         reinit
       else
         db_migrate_15
       fi;;
  esac
}

sec_db_remove () {
  if [ $POSTGRES -eq 1 ] ;
  then
    sql_pg "DROP SCHEMA IF EXISTS scap CASCADE;"
  else
    rm -f $SEC_DB
  fi
}

sec_db_exists () {
  if [ $POSTGRES -eq 1 ] ;
  then
    EXISTS=`sql_pg "SELECT exists (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'scap');" 2>/tmp/ERR | tr -d '\n\r' || echo 0`
    if [ "$EXISTS" = "f" ] ;
    then
      return 1;
    fi
    return 0;
  elif [ -f "$SEC_DB" ]
  then
    return 0;
  fi
  return 1;
}

init_sec_db_update () {
  if sec_db_exists; then
    check_db_version
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      CVE_REFDATE=`sql "SELECT max(modification_time) from ${SCHEMA}cves;"`
      test_sql_exit "Could not get reference date for CVEs and CPEs"
    done
  else
    echo "[i] Initializing scap database"
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql < $SCAP_RES_DIR/scap_db_init.sql
      test_sql_exit "Could not initialize SCAP database"
    done
    DB_LASTUPDATE=0
    CVE_REFDATE=0
    OVAL_REFDATE=0
  fi

  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    DB_LASTUPDATE=`sql "select value from ${SCHEMA}meta where name = 'last_update';"`
    test_sql_exit "Could not get last_update timestamp from database"
  done

  if [ -z "$CVE_REFDATE" ]
  then
    CVE_REFDATE=0
  fi

  if [ -z "$DB_LASTUPDATE" ]
  then
    # Happens when initial sync was aborted
    echo "[e] Error: Inconsistent data. Resetting SCAP database."
    sec_db_remove
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql < $SCAP_RES_DIR/scap_db_init.sql
      test_sql_exit "Could not reinitialize SCAP database"
    done
    CVE_REFDATE=0
    OVAL_REFDATE=0
    DB_LASTUPDATE=0
  fi

  updated_cpes=0
  updated_cves=0
  updated_ovaldefs=0
}

list_oval_files_sorted () {
for line in `list_oval_files_sortable "$1" | sort`
  do
    echo ${line#????????}
  done
}

list_oval_files_sortable () {
for ovalfile in `find "$1" -name "*.xml" -type f`
  do
    timestamp_raw=""
    timestamp_raw=`xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" 2>/dev/null`

    if [ -n "$timestamp_raw" ] \
       && [ -n `echo "$timestamp_raw" | egrep -x "[0-9]{4}\-(0[0-9]|1[012])\-([0-2][0-9]|3[01])[[:alnum:]]*"` ]
    then
      timestamp="`echo "$timestamp_raw" | sed -e 's/T.*//' | tr -d "-"`"
      echo "$timestamp$ovalfile"
    fi
  done
}

update_cvss () {
  if [ 0 -ne $updated_cves ] || [ 0 -ne $updated_cpes ]
  then
    echo "[i] Updating CVSS scores and CVE counts for CPEs"
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql "
      ${RECURSIVE_TRIGGERS_OFF}
      UPDATE ${SCHEMA}cpes
        SET max_cvss =
            (
              SELECT max(cvss)
              FROM ${SCHEMA}cves
              WHERE id IN
                (
                  SELECT cve
                  FROM ${SCHEMA}affected_products
                  WHERE cpe=cpes.id
                )
            ),
          cve_refs =
            (
              SELECT count(cve)
              FROM ${SCHEMA}affected_products
              WHERE cpe=cpes.id
            );
      "
      test_sql_exit "Could not update CVSS scores for CPEs"
    done
  else
    echo "[i] No CPEs or CVEs updated, skipping CVSS and CVE recount for CPEs."
  fi

  if [ 0 -ne $updated_cves ] || [ 0 -ne $updated_ovaldefs ]
  then
    echo "[i] Updating CVSS scores for OVAL definitions"
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql "
      ${RECURSIVE_TRIGGERS_OFF}
      UPDATE ${SCHEMA}ovaldefs
        SET max_cvss =
            (
              SELECT max(cvss)
              FROM ${SCHEMA}cves
              WHERE id IN
                (
                  SELECT cve
                  FROM ${SCHEMA}affected_ovaldefs
                  WHERE ovaldef=ovaldefs.id
                )
                AND cvss != 0.0
            );
      "
      test_sql_exit "Could not update CVSS scores for OVAL definitions"
    done
  else
    echo "[i] No OVAL definitions or CVEs updated, skipping CVSS recount for OVAL definitions."
  fi
}

update_placeholders () {
  if [ 0 -ne $updated_cves ]
  then
    echo "[i] Updating placeholder CPEs"
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql "
      UPDATE ${SCHEMA}cpes
        SET creation_time =
            (
              SELECT min(creation_time)
              FROM ${SCHEMA}cves
              WHERE id IN
                (
                  SELECT cve
                  FROM ${SCHEMA}affected_products
                  WHERE cpe=cpes.id
                )
            ),
            modification_time =
            (
              SELECT min(creation_time)
              FROM ${SCHEMA}cves
              WHERE id IN
                (
                  SELECT cve
                  FROM ${SCHEMA}affected_products
                  WHERE cpe=cpes.id
                )
            )
        WHERE cpes.title IS NULL
        AND (cpes.creation_time IS NULL OR cpes.modification_time IS NULL);
      "
      test_sql_exit "Could not update placeholder CPEs"
    done
  else
    echo "[i] No CVEs updated, skipping placeholder CPE update."
  fi
}

update_sec_db_private () {
  SCAP_PRIVATE_DIR="$SCAP_DIR/$PRIVATE_SUBDIR"
  if [ -d "$SCAP_PRIVATE_DIR" ]
  then
    echo "[i] Updating user defined data"

    echo "[i] Updating user OVAL definitions"
    xmlcount=0
    xmlcount=$(find "$SCAP_PRIVATE_DIR/oval/" -name "*.xml" -type f 2> /dev/null | wc -l)
    if [ -n "$xmlcount" ] && [ $xmlcount -ne 0 ]
    then
      if [ -z "$oval_files_sorted" ]
      then
        oval_files_sorted=`list_oval_files_sorted "$SCAP_DIR/oval/"`
        ovaldir_xml_files=`find "$SCAP_DIR/oval/" -name "*.xml" -type f 2> /dev/null`

        for xml_file in $ovaldir_xml_files
        do
          if [ -z `echo "$oval_files_sorted" | grep -x "$xml_file"` ]
          then
            echo "[w] Cannot find valid OVAL timestamp in '$xml_file'."
          fi
        done
      fi

      oval_files_sorted_private=`list_oval_files_sorted "$SCAP_PRIVATE_DIR/oval/"`
      ovaldir_xml_files=`find "$SCAP_PRIVATE_DIR/oval/" -name "*.xml" -type f 2> /dev/null`

      for xml_file in $ovaldir_xml_files
      do
        if [ -z `echo "$oval_files_sorted_private" | grep -x "$xml_file"` ]
        then
          echo "[w] Cannot find valid OVAL timestamp in '$xml_file'."
        fi
      done

      for non_xml_file in `find "$SCAP_PRIVATE_DIR/oval/" -type f -and -not -name "*.xml" 2> /dev/null`
      do
        if [ "${non_xml_file%%.asc}" = "${non_xml_file}" ] || ! [ -f "${non_xml_file%%.asc}" ]
        then
          echo "[w] Found non-XML and non-signature file '$non_xml_file'."
        fi
      done

      if [ -n "$oval_files_sorted" ]
      then
        for ovalfile in $oval_files_sorted_private
        do
          filedate=`stat -c "%Y" "$ovalfile" | cut -d " " -f 1 | tr -d "-"`
          filedate=$(( $filedate - ( $filedate % 60 ) ))
          if [ $filedate -gt $DB_LASTUPDATE ] || [ 1 = "$REBUILD_OVAL" ]
          then
            oval_timestamp=`xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" | date "+%s" -f -`

            if [ 1 = "$REBUILD_OVAL" ]
            then
              OVAL_REFDATE=0
            else
              reset_sql_tries
              until [ "$try_sql" -eq 0 ]
              do
                OVAL_REFDATE=`sql "SELECT max(modification_time) FROM ${SCHEMA}ovaldefs WHERE xml_file = '${ovalfile##$SCAP_DIR/}';"`
                test_sql_exit "Could not get OVAL reference date"
              done
              if [ -z "$OVAL_REFDATE" ]
              then
                OVAL_REFDATE=0
              fi
            fi

            if [ $oval_timestamp -gt $OVAL_REFDATE ]
            then
              validation_message=`xsltproc "$SCAP_RES_DIR/oval_verify.xsl" "$ovalfile" 2>/dev/null`
              validation_return="$?"
              if [ 0 -ne $validation_return ] && [ -n "$validation_message" ]
              then
                if [ 10 -eq $validation_return ]
                then
                  echo "[w] Validation of file '$ovalfile' failed: $validation_message"
                else
                  echo "[w] Error parsing '$ovalfile'."
                fi
              else
                if ! [ "$validation_message" = "file valid" ]
                then
                  echo "[w] Validation of file '$ovalfile' failed: $validation_message"
                else
                  echo "[i] Updating $ovalfile"
                  reset_sql_tries
                  until [ "$try_sql" -eq 0 ]
                  do
                    exec 4>&1
                    exit_codes=$(
                      (
                        set_interrupt_trap
                        (xsltproc --stringparam filename "${ovalfile##$SCAP_DIR/}" --stringparam refdate "$OVAL_REFDATE" "$SCAP_RES_DIR/oval_update.xsl" "$ovalfile" || echo "xsltproc:$?" >&3) | \
                        (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3)
                      ) 3>&1 >&4
                    )
                    exec 4>&-
                    test_exit_codes "Update of OVAL definitions failed at file '$ovalfile'"
                  done
                  updated_ovaldefs=1
                fi
              fi
            else
              echo "[i] Skipping $ovalfile, file has older timestamp than latest OVAL definition in database."
            fi
          else
            echo "[i] Skipping $ovalfile, file is older than last revision."
          fi
        done
      fi
    else
      echo "[i] No user defined OVAL files found"
    fi

    echo "[i] Cleaning up user OVAL data"
    DIR_STR_LENGTH=$((`echo "$SCAP_DIR" | wc -c` + 1))

    oval_files_shortened=""
    if [ 0 != "$xmlcount" ]
    then
      for ovalfile in $oval_files_sorted_private
      do
        oval_files_shortened=$oval_files_shortened"', '"`echo $ovalfile | cut -c ${DIR_STR_LENGTH}- `
      done
      oval_files_shortened=${oval_files_shortened##"', "}"'"
    fi

    oval_files_clause=""
    if [ ! -z "$oval_files_shortened" ]
    then
      oval_files_clause="AND (xml_file NOT IN ($oval_files_shortened))"
    fi

    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      deleted=`sql "SELECT DISTINCT xml_file FROM ${SCHEMA}ovaldefs WHERE (xml_file NOT LIKE 'oval/%') $oval_files_clause;"`
      test_sql_exit "Could not get list of deleted OVAL XML files"
    done

    if [ -n "$deleted" ]
    then
      echo "[i] Removing definitions formerly inserted from:"
      echo "$deleted"
      reset_sql_tries
      until [ "$try_sql" -eq 0 ]
      do
        sql "DELETE FROM ${SCHEMA}ovaldefs WHERE (xml_file NOT LIKE 'oval/%') $oval_files_clause;"
        test_sql_exit "Could not remove deleted OVAL definitions from database"
      done
    fi

    # TODO: This is not quite accurate as it uses the timestamp of the non-private data.
    LAST_UPDATE_TIMESTAMP=`sed 's/^\(.\{8\}\)/\1 /' $TIMESTAMP | date +%s -f -`

    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql "UPDATE ${SCHEMA}meta SET value ='$LAST_UPDATE_TIMESTAMP' WHERE name = 'last_update';"
      test_sql_exit "Could not update last_update timestamp in database"
    done
  else
    echo "[i] No user data directory '$SCAP_PRIVATE_DIR' found."
  fi
}

update_sec_db () {
  init_sec_db_update

  CPEBASE="$SCAP_DIR/official-cpe-dictionary_v2.2.xml"
  if [ -e $CPEBASE ]
  then
    filedate=`stat -c "%Y" "$CPEBASE" | cut -d " " -f 1 | tr -d "-"`
    filedate=$(( $filedate - ( $filedate % 60 ) ))
    if [ $filedate -gt $DB_LASTUPDATE ]
    then
      echo "[i] Updating CPEs"

      filesize=`stat -c "%s" "$CPEBASE"`
      if [ "0" -ne "$SPLIT_PART_SIZE" ] && [ "$filesize" -gt $(($SPLIT_PART_SIZE * 1024))  ]
      then
        echo "[i] File is larger than ${SPLIT_PART_SIZE}k. Splitting into multiple parts"
        "$SCAP_RES_DIR/xml_split" "$CPEBASE" -M "$SPLIT_PART_SIZE" -o "$SYNC_TMP_DIR"
        cpefile_nodir=${CPEBASE##*/}
        for cpepart in `find "$SYNC_TMP_DIR" -path "$SYNC_TMP_DIR/${cpefile_nodir%%.xml}.*.xml" 2> /dev/null | sort`
        do
          echo "[i] Updating part $cpepart"
          reset_sql_tries
          until [ "$try_sql" -eq 0 ]
          do
            exec 4>&1
            exit_codes=$(
              (
                set_interrupt_trap
                (sed 's/&/&amp;/g' $cpepart || echo "sed:$?" >&3 ) | \
                (xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cpe_youngerthan.xsl - || echo "xsltproc:$?" >&3) | \
                (xsltproc $SCAP_RES_DIR/cpe_update.xsl - || echo "xsltproc:$?" >&3) | \
                (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3) \
              ) 3>&1 >&4
            )
            exec 4>&-
            test_exit_codes "Update of CPEs failed"
          done
        done
      else
        reset_sql_tries
        until [ "$try_sql" -eq 0 ]
        do
          exec 4>&1
          exit_codes=$(
            (
              set_interrupt_trap
              (sed 's/&/&amp;/g' $CPEBASE || echo "sed:$?" >&3 ) | \
              (xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cpe_youngerthan.xsl - || echo "xsltproc:$?" >&3) | \
              (xsltproc $SCAP_RES_DIR/cpe_update.xsl - || echo "xsltproc:$?" >&3) | \
              (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3) \
            ) 3>&1 >&4
          )
          exec 4>&-
          test_exit_codes "Update of CPEs failed"
        done
      fi
      updated_cpes=1
    else
      echo "[i] Skipping CPEs, file is older than last revision"
    fi
  else
    echo "[w] No CPE dictionary found in $SCAP_DIR"
  fi

  xmlcount=$(ls $SCAP_DIR/nvdcve-2.0-*.xml 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then
    for cvefile in `ls $SCAP_DIR/nvdcve-2.0-*.xml`
    do
      filedate=`stat -c "%Y" "$cvefile" | cut -d " " -f 1 | tr -d "-"`
      filedate=$(( $filedate - ( $filedate % 60 ) ))
      if [ $filedate -gt $DB_LASTUPDATE ]
      then
        echo "[i] Updating $cvefile"

        filesize=`stat -c "%s" "$cvefile"`
        if [ "0" -ne "$SPLIT_PART_SIZE" ] && [ "$filesize" -gt $(($SPLIT_PART_SIZE * 1024))  ]
        then
          echo "[i] File is larger than ${SPLIT_PART_SIZE}k. Splitting into multiple parts"
          "$SCAP_RES_DIR/xml_split" "$cvefile" -M "$SPLIT_PART_SIZE" -o "$SYNC_TMP_DIR"
          cvefile_nodir=${cvefile##*/}
          for cvepart in `find "$SYNC_TMP_DIR" -path "$SYNC_TMP_DIR/${cvefile_nodir%%.xml}.*.xml" 2> /dev/null | sort`
          do
            echo "[i] Updating part $cvepart"
            reset_sql_tries
            until [ "$try_sql" -eq 0 ]
            do
              exec 4>&1
              exit_codes=$(
                (
                  set_interrupt_trap
                  (xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cve_youngerthan.xsl $cvepart || echo "xsltproc:$?" >&3) | \
                  (xsltproc $SCAP_RES_DIR/cve_update.xsl - || echo "xsltproc:$?" >&3) | \
                  (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3)
                ) 3>&1 >&4
              )
              exec 4>&-
              rm "$cvepart"
              test_exit_codes "Update of CVEs failed at part '$cvepart'"
            done
          done
        else
          reset_sql_tries
          until [ "$try_sql" -eq 0 ]
          do
            exec 4>&1
            exit_codes=$(
              (
                set_interrupt_trap
                (xsltproc --stringparam refdate $CVE_REFDATE $SCAP_RES_DIR/cve_youngerthan.xsl $cvefile || echo "xsltproc:$?" >&3) | \
                (xsltproc $SCAP_RES_DIR/cve_update.xsl - || echo "xsltproc:$?" >&3) | \
                (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3)
              ) 3>&1 >&4
            )
            exec 4>&-
            test_exit_codes "Update of CVEs failed at file '$cvefile'"
          done
        fi
        updated_cves=1
      else
        echo "[i] Skipping $cvefile, file is older than last revision"
      fi
    done
  else
    echo "[w] No CVEs found in $SCAP_DIR"
  fi

  echo "[i] Updating OVAL data"
  xmlcount=$(find "$SCAP_DIR/oval/" -name "*.xml" -type f 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then

    oval_files_sorted=`list_oval_files_sorted "$SCAP_DIR/oval/"`
    ovaldir_xml_files=`find "$SCAP_DIR/oval/" -name "*.xml" -type f 2> /dev/null`

    for xml_file in $ovaldir_xml_files
    do
      if [ -z `echo "$oval_files_sorted" | grep -x "$xml_file"` ]
      then
        echo "[w] Cannot find valid OVAL timestamp in '$xml_file'."
      fi
    done

    for ovalfile in $oval_files_sorted
    do
      filedate=`stat -c "%Y" "$ovalfile" | cut -d " " -f 1 | tr -d "-"`
      filedate=$(( $filedate - ( $filedate % 60 ) ))
      if [ $filedate -gt $DB_LASTUPDATE ] || [ 1 = "$REBUILD_OVAL" ]
      then
        oval_timestamp=`xsltproc "$SCAP_RES_DIR/oval_timestamp.xsl" "$ovalfile" | date "+%s" -f -`

        if [ 1 = "$REBUILD_OVAL" ]
        then
          OVAL_REFDATE=0
        else
          reset_sql_tries
          until [ "$try_sql" -eq 0 ]
          do
            OVAL_REFDATE=`sql "SELECT max(modification_time) FROM ${SCHEMA}ovaldefs WHERE xml_file = '${ovalfile##$SCAP_DIR/}';"`
            test_sql_exit "Could not get OVAL reference date from database"
          done
          if [ -z "$OVAL_REFDATE" ]
          then
            OVAL_REFDATE=0
          fi
        fi

        if [ $oval_timestamp -gt $OVAL_REFDATE ]
        then
          echo "[i] Updating $ovalfile"
          reset_sql_tries
          until [ "$try_sql" -eq 0 ]
          do
            exec 4>&1
            exit_codes=$(
              (
                set_interrupt_trap
                (xsltproc --stringparam filename "${ovalfile##$SCAP_DIR/}" --stringparam refdate "$OVAL_REFDATE" "$SCAP_RES_DIR/oval_update.xsl" "$ovalfile" || echo "xsltproc:$?" >&3) | \
                (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3)
              ) 3>&1 >&4
            )
            exec 4>&-
            test_exit_codes "Update of OVAL definitions failed at file '$ovalfile'"
          done
          updated_ovaldefs=1
        else
          echo "[i] Skipping $ovalfile, file has older timestamp than latest OVAL definition in database."
        fi
      else
        echo "[i] Skipping $ovalfile, file is older than last revision."
      fi
    done
  else
    echo "[w] No XML files found in $SCAP_DIR/oval/."
  fi

  update_sec_db_private
  update_cvss
  update_placeholders

  LAST_UPDATE_TIMESTAMP=`sed 's/^\(.\{8\}\)/\1 /' $TIMESTAMP | env TZ="UTC" date +%s -f -`
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    sql "UPDATE ${SCHEMA}meta SET value ='$LAST_UPDATE_TIMESTAMP' WHERE name = 'last_update';"
    test_sql_exit "Could not update last_update timestamp in database"
  done
}

do_refresh () {
  echo "[i] Refreshing database without feed sync."
  update_sec_db
}

do_refresh_private () {
  REFRESH_PRIVATE_ONLY=1
  init_sec_db_update
  update_sec_db_private
  update_cvss
}

psql_setup "tasks"
if [ -n "$1" ]; then
  while test $# -gt 0; do
    case "$1" in
      --help)
        do_help
        exit 0
        ;;
      --rsync)
        do_rsync
        exit 0
        ;;
      --wget)
        do_wget
        exit 0
        ;;
      --curl)
        do_curl
        exit 0
        ;;
      --database)
		if [ $POSTGRES -eq 1 ] ;
		then
		  shift
          if [ $# -gt 0 ]; then
		    psql_setup $1
	      else
			echo "[e] --database argument missing"
			exit 1
		  fi
		else
		  echo "[e] --database option available only for Postgres"
		  exit 1
		fi
        ;;
      --verbose)
        VERBOSE=1
        ;;
      --refresh)
        do_refresh
        exit 0
        ;;
      --migrate)
        do_refresh
        exit 0
        ;;
      --refresh-private)
        do_refresh_private
        exit 0
        ;;
      --check)
        exit 0
        ;;
      --version)
        echo $VERSION
        exit 0
        ;;
      --identify)
        echo "SCAPSYNC|$SCRIPT_NAME|$VERSION|$FEED_NAME|$RESTRICTED|SCAPSYNC"
        exit 0
        ;;
      --selftest)
        SELFTEST_FAIL=0
        do_self_test
        exit $SELFTEST_FAIL
        ;;
      --describe)
        do_describe
        exit 0
        ;;
      --feedversion)
        do_feedversion
        exit 0
        ;;
    esac
    shift
  done
fi

show_intro
do_sync
update_sec_db

exit 0