From 3fbd791dfb80dd638bbe902b0a31d55b7fd4be42 Mon Sep 17 00:00:00 2001 From: Eliezer Croitoru Date: Sun, 18 Jul 2021 23:13:59 +0300 Subject: [PATCH] 37 --- collect-clish-scripts-daemon.sh | 17 ++-- cron-example-with-diff-dstdom.sh | 152 +++++++++++++++++++------------ 2 files changed, 99 insertions(+), 70 deletions(-) diff --git a/collect-clish-scripts-daemon.sh b/collect-clish-scripts-daemon.sh index e5e1eaf..729a35d 100755 --- a/collect-clish-scripts-daemon.sh +++ b/collect-clish-scripts-daemon.sh @@ -25,12 +25,9 @@ fi FILENAME="/storage/collect-clish-scripts.sh" -which curl_cli >/dev/null 2>&1 && CURL="curl_cli" -which curl >/dev/null 2>&1 && CURL="curl" +CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt" +SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}" -export CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt" -export SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}" -alias curl_cli="curl_cli --cacert ${CA_CERT_BUNDLE_PATH}" function collect() { START_EXECUTION_TIME=$( date +"%Y_%m_%d_%H_%M_%SS" ) @@ -86,7 +83,7 @@ function collect() { echo "Starting to work on: \"${file}\" at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |tee -a "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/execution.log" echo "Starting to work on: \"${file}\" at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |logger - clish -f "${file}" + su - "admin" -c "/pfrm2.0/bin/clish -f \"${file}\"" mv -v "${file}" "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/${count}.clish_${START_EXECUTION_TIME}" echo "Finished working on: \"${file}\" , Exit Code: $? , at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |tee -a "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/execution.log" echo "Finished working on: \"${file}\" , Exit Code: $? , at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |logger @@ -111,16 +108,16 @@ do if [ "${RUN_AS_A_FUNCTION}" -eq "1" ];then collect else - /bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1 + su - admin -c "/bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1" fi sleep 5 continue fi else - REMOTE_ETAG=$(${CURL} -s -I "${URL}" |grep "Etag" -i |head -1 |awk '{print $2}'|sed -e "s@\"@@") + REMOTE_ETAG=$( curl_cli -s --cacert "${SSL_CERT_FILE}" -I "${URL}" |grep "Etag" -i |head -1 |awk '{print $2}'|sed -e "s@\"@@" ) if [ "${CURRENT_ETAG}" != "${REMOTE_ETAG}" ];then - ${CURL} -s "${URL}" -o "${TMP_REMOTE_IN_FILE}" + curl_cli -s --cacert "${SSL_CERT_FILE}""${URL}" -o "${TMP_REMOTE_IN_FILE}" REMOTE_MD5=$( md5sum "${TMP_REMOTE_IN_FILE}" |awk '{print $1}' ) fi @@ -134,7 +131,7 @@ do if [ "${RUN_AS_A_FUNCTION}" -eq "1" ];then collect else - /bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1 + su - admin -c "/bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1" fi sleep 5 diff --git a/cron-example-with-diff-dstdom.sh b/cron-example-with-diff-dstdom.sh index a7c05f7..27d04f9 100755 --- a/cron-example-with-diff-dstdom.sh +++ b/cron-example-with-diff-dstdom.sh @@ -1,6 +1,25 @@ #!/usr/bin/env bash +FORCE_ADMIN_USER="1" + +if [ "${FORCE_ADMIN_USER}" -eq "1" ];then + if [ "${USER}" != "admin" ]; then + echo "Switching to admin user" + exec su "admin" "$0" -- "$@" + exit $? + fi +fi + FLAGS_PREFIX="/tmp/dst-dom-script-flag_" +WORK_DIR="/tmp/dst-dom-script" + +mkdir -vp "${WORK_DIR}" +cd "${WORK_DIR}" + +export CPDIR=/opt/fw1 +export FWDIR=${CPDIR} +export SUROOT=/var/suroot +. /pfrm2.0/etc/bashrc if [ -f "${FLAGS_PREFIX}unsetx" ];then set -x @@ -13,79 +32,79 @@ REGEX_FLAG_ENABLED="0" LOCK_FILE="/tmp/dst-domain-cron-lockfile" DUMP_ENV="/tmp/env_cd5fecd5-7123-4a21-bd02-242f1d695a6d" -export CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt" -export SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}" -#alias curl_cli="curl_cli --cacert ${CA_CERT_BUNDLE_PATH}" +CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt" +SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}" if [ -f "${LOCK_FILE}" ];then - echo "Lockfile exits, stopping update" - exit 0 + echo "Lockfile \"${LOCK_FILE}\" exits, stopping update" + exit 0 fi touch "${LOCK_FILE}" if [ -f "${FLAGS_PREFIX}debug" ];then - DEBUG="1" + DEBUG="1" fi if [ -f "${FLAGS_PREFIX}dry-run" ];then - DRY_RUN="1" + DRY_RUN="1" + echo "Runnning dry run" fi if [ -f "${FLAGS_PREFIX}cleanup-after" ];then - CLEANUP_AFTER="1" + CLEANUP_AFTER="1" fi if [ -f "${FLAGS_PREFIX}dont-cleanup-after" ];then - CLEANUP_AFTER="0" + CLEANUP_AFTER="0" fi if [ -f "${FLAGS_PREFIX}add-regex-flag" ];then - REGEX_FLAG_ENABLED="1" + REGEX_FLAG_ENABLED="1" fi if [ -f "${FLAGS_PREFIX}dump-env" ];then - env |tee "${DUMP_ENV}" - export | tee -a "${DUMP_ENV}" + env |tee "${DUMP_ENV}" + export | tee -a "${DUMP_ENV}" fi function dstdomain_to_regex() { - prefix="\." - suffix="\." - dot="\." - dash="-" + prefix="\." + suffix="\." + dot="\." + dash="-" - domain="$1" - dstdomain="0" - dotsuffix="0" + domain="$1" + dstdomain="0" + dotsuffix="0" - echo "${domain}" | grep -e "^\." > /dev/null - if [ "$?" -eq "0" ];then - dstdomain=1 - fi + echo "${domain}" | grep -e "^\." > /dev/null + if [ "$?" -eq "0" ];then + dstdomain=1 + fi - echo "${domain}" | grep -e "\.$" > /dev/null - if [ "$?" -eq "0" ];then - dotsuffix=1 - fi + echo "${domain}" | grep -e "\.$" > /dev/null + if [ "$?" -eq "0" ];then + dotsuffix=1 + fi - case ${dstdomain} in - 1) - echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/" - echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\$/" + case ${dstdomain} in + 1) + echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/" + echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\$/" - echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/" - echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\\\.\\$/" + echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/" + echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\\\.\\$/" - ;; - *) - echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/" - echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/" + ;; + *) + echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/" + echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/" - ;; - esac + ;; + esac } @@ -95,14 +114,14 @@ APP_NAME="$1" URL="$2" if [ -f "dst-domain-url" ];then - echo "Overriding URL with a local dst-domain-url file" - DST_DOM_URL_FILE_SIZE=$(cat dst-domain-url |wc -l) - if [ "${DST_DOM_URL_FILE_SIZE}" -gt "0" ];then - URl=$( head -n1 dst-domain-url ) - else - echo "dst-domain-url is empty" - exit 1 - fi + echo "Overriding URL with a local dst-domain-url file" + DST_DOM_URL_FILE_SIZE=$(cat dst-domain-url |wc -l) + if [ "${DST_DOM_URL_FILE_SIZE}" -gt "0" ];then + URl=$( head -n1 dst-domain-url ) + else + echo "dst-domain-url is empty" + exit 1 + fi fi if [ -z "${APP_NAME}" ];then @@ -121,9 +140,8 @@ fi TMP_DOWNLOAD_FILE=$(mktemp) -/opt/fw1/bin/curl_cli --cacert "${CA_CERT_BUNDLE_PATH}" -s "${URL}" -o "${TMP_DOWNLOAD_FILE}" -RES=$? - +/opt/fw1/bin/curl_cli -s --cacert "${SSL_CERT_FILE}" "${URL}" -o "${TMP_DOWNLOAD_FILE}" >> /tmp/log.1 +RES="$?" if [ "${RES}" -gt "0" ];then echo "Error Downloading file from URL: \"${URL}\"" logger "Error Downloading file from URL: \"${URL}\"" @@ -140,11 +158,13 @@ TMP_CLISH_TRANSACTION_FILE=$( mktemp ) TMP_DIFF_FILE=$( mktemp ) -#clish -c "show configuration"|egrep "^set application application-name \"${APP_NAME}\"" > ${TMP_CURRENT_CONFIG_FILE} - TMP_CURRENT_APP_CONTENT_FILE=$( mktemp ) -clish -c "show application application-name \"${APP_NAME}\"" | sed -e "s@^description.*@@g" \ +APP_DETAILS=$( clish -c "show application application-name ${APP_NAME}" ) +echo "$? exit code from clish -c \"show application application-name ${APP_NAME}\"" + +echo "${APP_DETAILS}" | sed -e "s@^description.*@@g" \ + -e "s@.*Role\ is\ not\ assigned\ to\ user.*@@g" \ -e "s@^application\-name\:.*@@g" \ -e "s@^application\-id\:.*@@g" \ -e "s@^Categories\:.*@@g" \ @@ -152,8 +172,12 @@ clish -c "show application application-name \"${APP_NAME}\"" | sed -e "s@^descri -e 's@^[ \t]\+@@g' \ -e '/^$/ d' > ${TMP_CURRENT_APP_CONTENT_FILE} -CURRENT_APP_CONTENT_REGEX=$( cat ${TMP_CURRENT_APP_CONTENT_FILE} |sort ) +CURRENT_APP_CONTENT_REGEX=$( cat "${TMP_CURRENT_APP_CONTENT_FILE}" |sort ) REMOTE_APP_CONTENT_REGEX=$( mktemp ) +echo "${CURRENT_APP_CONTENT_REGEX}" > "${TMP_CURRENT_APP_CONTENT_FILE}" + +comp_start=`date +%s` +echo "Compiling APP_REGEX started at: ${comp_start}" while IFS= read -r line do @@ -163,11 +187,16 @@ do fi dstdomain_to_regex_result="$(dstdomain_to_regex ${line})" - echo "${dstdomain_to_regex_result}" |tee -a "${REMOTE_APP_CONTENT_REGEX}" >/dev/null + echo "${dstdomain_to_regex_result}" |tee -a "${REMOTE_APP_CONTENT_REGEX}" >/dev/null done < ${TMP_DOWNLOAD_FILE} -SORTED_REMOTE_APP_CONTENT_REGEX=$( cat ${REMOTE_APP_CONTENT_REGEX}| sort| uniq ) +comp_end=`date +%s` +comp_runtime=$( echo "$comp_end - $comp_start" | bc -l ) + +echo "FINISHED Compiling APP_REGEX finished at: ${comp_end} , Took: ${comp_runtime} Seconds" + +SORTED_REMOTE_APP_CONTENT_REGEX=$( cat "${REMOTE_APP_CONTENT_REGEX}" | sort| uniq ) echo "${SORTED_REMOTE_APP_CONTENT_REGEX}" > "${REMOTE_APP_CONTENT_REGEX}" DIFF=$( diff "${TMP_CURRENT_APP_CONTENT_FILE}" "${REMOTE_APP_CONTENT_REGEX}" |sed -e "1,3d;" ) @@ -191,6 +220,9 @@ sed -i -e 's@\\@\\\\\\@g' "${TMP_CLISH_TRANSACTION_FILE}" if [ "${DRY_RUN}" -eq "0" ];then clish -f "${TMP_CLISH_TRANSACTION_FILE}" + echo "$? exit code from clish -f" +else + echo "Running in DRY-RUN MODE" fi echo "Finished Transaction" @@ -199,8 +231,8 @@ echo "Cleaning up files ..." if [ "${CLEANUP_AFTER}" -eq "1" ];then rm -v "${TMP_DOWNLOAD_FILE}" rm -v "${TMP_CLISH_UPDATE_FILE}" - rm -v "${TMP_CURRENT_APP_CONTENT_FILE}" - rm -v "${REMOTE_APP_CONTENT_REGEX}" + rm -v "${TMP_CURRENT_APP_CONTENT_FILE}" + rm -v "${REMOTE_APP_CONTENT_REGEX}" rm -v "${TMP_CURRENT_CONFIG_FILE}" rm -v "${TMP_DIFF_FILE}" rm -v "${TMP_CLISH_TRANSACTION_FILE}" @@ -209,8 +241,8 @@ else echo "Don't forget to cleanup the files:" echo "${TMP_DOWNLOAD_FILE}" echo "${TMP_CLISH_UPDATE_FILE}" - echo "${TMP_CURRENT_APP_CONTENT_FILE}" - echo "${REMOTE_APP_CONTENT_REGEX}" + echo "${TMP_CURRENT_APP_CONTENT_FILE}" + echo "${REMOTE_APP_CONTENT_REGEX}" echo "${TMP_CURRENT_CONFIG_FILE}" echo "${TMP_DIFF_FILE}" echo "${TMP_CLISH_TRANSACTION_FILE}"