NgTech-LTD/cp-tls-inspection-application-update-cron
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

37

Browse Source
This commit is contained in:
Eliezer Croitoru 2021-07-18 23:13:59 +03:00
parent 0b7995bb9c
commit 3fbd791dfb
2 changed files with 99 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
collect-clish-scripts-daemon.sh
View File

@ -25,12 +25,9 @@ fi
FILENAME="/storage/collect-clish-scripts.sh" FILENAME="/storage/collect-clish-scripts.sh"
which curl_cli >/dev/null 2>&1 && CURL="curl_cli" CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt"
which curl >/dev/null 2>&1 && CURL="curl" SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}"
export CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt"
export SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}"
alias curl_cli="curl_cli --cacert ${CA_CERT_BUNDLE_PATH}"
function collect() { function collect() {
START_EXECUTION_TIME=$( date +"%Y_%m_%d_%H_%M_%SS" ) START_EXECUTION_TIME=$( date +"%Y_%m_%d_%H_%M_%SS" )
@ -86,7 +83,7 @@ function collect() {
echo "Starting to work on: \"${file}\" at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |tee -a "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/execution.log" echo "Starting to work on: \"${file}\" at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |tee -a "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/execution.log"
echo "Starting to work on: \"${file}\" at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |logger echo "Starting to work on: \"${file}\" at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |logger
clish -f "${file}" su - "admin" -c "/pfrm2.0/bin/clish -f \"${file}\""
mv -v "${file}" "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/${count}.clish_${START_EXECUTION_TIME}" mv -v "${file}" "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/${count}.clish_${START_EXECUTION_TIME}"
echo "Finished working on: \"${file}\" , Exit Code: $? , at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |tee -a "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/execution.log" echo "Finished working on: \"${file}\" , Exit Code: $? , at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |tee -a "${ARCHIVE_PATH}/${START_EXECUTION_DATE}/execution.log"
echo "Finished working on: \"${file}\" , Exit Code: $? , at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |logger echo "Finished working on: \"${file}\" , Exit Code: $? , at: $( date +"%Y_%m_%d_%H_%M_%SS" ) , Execution count: ${count}" |logger
@ -111,16 +108,16 @@ do
if [ "${RUN_AS_A_FUNCTION}" -eq "1" ];then if [ "${RUN_AS_A_FUNCTION}" -eq "1" ];then
collect collect
else else
/bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1 su - admin -c "/bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1"
fi fi
sleep 5 sleep 5
continue continue
fi fi
else else
REMOTE_ETAG=$(${CURL} -s -I "${URL}" |grep "Etag" -i |head -1 |awk '{print $2}'|sed -e "s@\"@@") REMOTE_ETAG=$( curl_cli -s --cacert "${SSL_CERT_FILE}" -I "${URL}" |grep "Etag" -i |head -1 |awk '{print $2}'|sed -e "s@\"@@" )
if [ "${CURRENT_ETAG}" != "${REMOTE_ETAG}" ];then if [ "${CURRENT_ETAG}" != "${REMOTE_ETAG}" ];then
${CURL} -s "${URL}" -o "${TMP_REMOTE_IN_FILE}" curl_cli -s --cacert "${SSL_CERT_FILE}""${URL}" -o "${TMP_REMOTE_IN_FILE}"
REMOTE_MD5=$( md5sum "${TMP_REMOTE_IN_FILE}" |awk '{print $1}' ) REMOTE_MD5=$( md5sum "${TMP_REMOTE_IN_FILE}" |awk '{print $1}' )
fi fi
@ -134,7 +131,7 @@ do
if [ "${RUN_AS_A_FUNCTION}" -eq "1" ];then if [ "${RUN_AS_A_FUNCTION}" -eq "1" ];then
collect collect
else else
/bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1 su - admin -c "/bin/bash /storage/collect-clish-scripts.sh >/dev/null 2>&1"
fi fi
sleep 5 sleep 5

152
cron-example-with-diff-dstdom.sh
View File

@ -1,6 +1,25 @@
#!/usr/bin/env bash #!/usr/bin/env bash
FORCE_ADMIN_USER="1"
if [ "${FORCE_ADMIN_USER}" -eq "1" ];then
if [ "${USER}" != "admin" ]; then
echo "Switching to admin user"
exec su "admin" "$0" -- "$@"
exit $?
fi
fi
FLAGS_PREFIX="/tmp/dst-dom-script-flag_" FLAGS_PREFIX="/tmp/dst-dom-script-flag_"
WORK_DIR="/tmp/dst-dom-script"
mkdir -vp "${WORK_DIR}"
cd "${WORK_DIR}"
export CPDIR=/opt/fw1
export FWDIR=${CPDIR}
export SUROOT=/var/suroot
. /pfrm2.0/etc/bashrc
if [ -f "${FLAGS_PREFIX}unsetx" ];then if [ -f "${FLAGS_PREFIX}unsetx" ];then
set -x set -x
@ -13,79 +32,79 @@ REGEX_FLAG_ENABLED="0"
LOCK_FILE="/tmp/dst-domain-cron-lockfile" LOCK_FILE="/tmp/dst-domain-cron-lockfile"
DUMP_ENV="/tmp/env_cd5fecd5-7123-4a21-bd02-242f1d695a6d" DUMP_ENV="/tmp/env_cd5fecd5-7123-4a21-bd02-242f1d695a6d"
export CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt" CA_CERT_BUNDLE_PATH="/pfrm2.0/opt/fw1/bin/ca-bundle.crt"
export SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}" SSL_CERT_FILE="${CA_CERT_BUNDLE_PATH}"
#alias curl_cli="curl_cli --cacert ${CA_CERT_BUNDLE_PATH}"
if [ -f "${LOCK_FILE}" ];then if [ -f "${LOCK_FILE}" ];then
echo "Lockfile exits, stopping update" echo "Lockfile \"${LOCK_FILE}\" exits, stopping update"
exit 0 exit 0
fi fi
touch "${LOCK_FILE}" touch "${LOCK_FILE}"
if [ -f "${FLAGS_PREFIX}debug" ];then if [ -f "${FLAGS_PREFIX}debug" ];then
DEBUG="1" DEBUG="1"
fi fi
if [ -f "${FLAGS_PREFIX}dry-run" ];then if [ -f "${FLAGS_PREFIX}dry-run" ];then
DRY_RUN="1" DRY_RUN="1"
echo "Runnning dry run"
fi fi
if [ -f "${FLAGS_PREFIX}cleanup-after" ];then if [ -f "${FLAGS_PREFIX}cleanup-after" ];then
CLEANUP_AFTER="1" CLEANUP_AFTER="1"
fi fi
if [ -f "${FLAGS_PREFIX}dont-cleanup-after" ];then if [ -f "${FLAGS_PREFIX}dont-cleanup-after" ];then
CLEANUP_AFTER="0" CLEANUP_AFTER="0"
fi fi
if [ -f "${FLAGS_PREFIX}add-regex-flag" ];then if [ -f "${FLAGS_PREFIX}add-regex-flag" ];then
REGEX_FLAG_ENABLED="1" REGEX_FLAG_ENABLED="1"
fi fi
if [ -f "${FLAGS_PREFIX}dump-env" ];then if [ -f "${FLAGS_PREFIX}dump-env" ];then
env |tee "${DUMP_ENV}" env |tee "${DUMP_ENV}"
export | tee -a "${DUMP_ENV}" export | tee -a "${DUMP_ENV}"
fi fi
function dstdomain_to_regex() { function dstdomain_to_regex() {
prefix="\." prefix="\."
suffix="\." suffix="\."
dot="\." dot="\."
dash="-" dash="-"
domain="$1" domain="$1"
dstdomain="0" dstdomain="0"
dotsuffix="0" dotsuffix="0"
echo "${domain}" | grep -e "^\." > /dev/null echo "${domain}" | grep -e "^\." > /dev/null
if [ "$?" -eq "0" ];then if [ "$?" -eq "0" ];then
dstdomain=1 dstdomain=1
fi fi
echo "${domain}" | grep -e "\.$" > /dev/null echo "${domain}" | grep -e "\.$" > /dev/null
if [ "$?" -eq "0" ];then if [ "$?" -eq "0" ];then
dotsuffix=1 dotsuffix=1
fi fi
case ${dstdomain} in case ${dstdomain} in
1) 1)
echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/" echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/"
echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\$/" echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\$/"
echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/" echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/"
echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\\\.\\$/" echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^[0-9a-zA-Z\\\-\\\.]+\\\.@g" -e "s/$/\\\\.\\$/"
;; ;;
*) *)
echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/" echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\$/"
echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/" echo "${domain}" | sed -e "s/^${prefix}//" -e "s/${suffix}$//" -e "s/${dash}/\\\-/g" -e "s/${dot}/\\\./g" -e "s@^@\\^@g" -e "s/$/\\\.\\$/"
;; ;;
esac esac
} }
@ -95,14 +114,14 @@ APP_NAME="$1"
URL="$2" URL="$2"
if [ -f "dst-domain-url" ];then if [ -f "dst-domain-url" ];then
echo "Overriding URL with a local dst-domain-url file" echo "Overriding URL with a local dst-domain-url file"
DST_DOM_URL_FILE_SIZE=$(cat dst-domain-url |wc -l) DST_DOM_URL_FILE_SIZE=$(cat dst-domain-url |wc -l)
if [ "${DST_DOM_URL_FILE_SIZE}" -gt "0" ];then if [ "${DST_DOM_URL_FILE_SIZE}" -gt "0" ];then
URl=$( head -n1 dst-domain-url ) URl=$( head -n1 dst-domain-url )
else else
echo "dst-domain-url is empty" echo "dst-domain-url is empty"
exit 1 exit 1
fi fi
fi fi
if [ -z "${APP_NAME}" ];then if [ -z "${APP_NAME}" ];then
@ -121,9 +140,8 @@ fi
TMP_DOWNLOAD_FILE=$(mktemp) TMP_DOWNLOAD_FILE=$(mktemp)
/opt/fw1/bin/curl_cli --cacert "${CA_CERT_BUNDLE_PATH}" -s "${URL}" -o "${TMP_DOWNLOAD_FILE}" /opt/fw1/bin/curl_cli -s --cacert "${SSL_CERT_FILE}" "${URL}" -o "${TMP_DOWNLOAD_FILE}" >> /tmp/log.1
RES=$? RES="$?"
if [ "${RES}" -gt "0" ];then if [ "${RES}" -gt "0" ];then
echo "Error Downloading file from URL: \"${URL}\"" echo "Error Downloading file from URL: \"${URL}\""
logger "Error Downloading file from URL: \"${URL}\"" logger "Error Downloading file from URL: \"${URL}\""
@ -140,11 +158,13 @@ TMP_CLISH_TRANSACTION_FILE=$( mktemp )
TMP_DIFF_FILE=$( mktemp ) TMP_DIFF_FILE=$( mktemp )
#clish -c "show configuration"|egrep "^set application application-name \"${APP_NAME}\"" > ${TMP_CURRENT_CONFIG_FILE}
TMP_CURRENT_APP_CONTENT_FILE=$( mktemp ) TMP_CURRENT_APP_CONTENT_FILE=$( mktemp )
clish -c "show application application-name \"${APP_NAME}\"" | sed -e "s@^description.*@@g" \ APP_DETAILS=$( clish -c "show application application-name ${APP_NAME}" )
echo "$? exit code from clish -c \"show application application-name ${APP_NAME}\""
echo "${APP_DETAILS}" | sed -e "s@^description.*@@g" \
-e "s@.*Role\ is\ not\ assigned\ to\ user.*@@g" \
-e "s@^application\-name\:.*@@g" \ -e "s@^application\-name\:.*@@g" \
-e "s@^application\-id\:.*@@g" \ -e "s@^application\-id\:.*@@g" \
-e "s@^Categories\:.*@@g" \ -e "s@^Categories\:.*@@g" \
@ -152,8 +172,12 @@ clish -c "show application application-name \"${APP_NAME}\"" | sed -e "s@^descri
-e 's@^[ \t]\+@@g' \ -e 's@^[ \t]\+@@g' \
-e '/^$/ d' > ${TMP_CURRENT_APP_CONTENT_FILE} -e '/^$/ d' > ${TMP_CURRENT_APP_CONTENT_FILE}
CURRENT_APP_CONTENT_REGEX=$( cat ${TMP_CURRENT_APP_CONTENT_FILE} |sort ) CURRENT_APP_CONTENT_REGEX=$( cat "${TMP_CURRENT_APP_CONTENT_FILE}" |sort )
REMOTE_APP_CONTENT_REGEX=$( mktemp ) REMOTE_APP_CONTENT_REGEX=$( mktemp )
echo "${CURRENT_APP_CONTENT_REGEX}" > "${TMP_CURRENT_APP_CONTENT_FILE}"
comp_start=`date +%s`
echo "Compiling APP_REGEX started at: ${comp_start}"
while IFS= read -r line while IFS= read -r line
do do
@ -163,11 +187,16 @@ do
fi fi
dstdomain_to_regex_result="$(dstdomain_to_regex ${line})" dstdomain_to_regex_result="$(dstdomain_to_regex ${line})"
echo "${dstdomain_to_regex_result}" |tee -a "${REMOTE_APP_CONTENT_REGEX}" >/dev/null echo "${dstdomain_to_regex_result}" |tee -a "${REMOTE_APP_CONTENT_REGEX}" >/dev/null
done < ${TMP_DOWNLOAD_FILE} done < ${TMP_DOWNLOAD_FILE}
SORTED_REMOTE_APP_CONTENT_REGEX=$( cat ${REMOTE_APP_CONTENT_REGEX}| sort| uniq ) comp_end=`date +%s`
comp_runtime=$( echo "$comp_end - $comp_start" | bc -l )
echo "FINISHED Compiling APP_REGEX finished at: ${comp_end} , Took: ${comp_runtime} Seconds"
SORTED_REMOTE_APP_CONTENT_REGEX=$( cat "${REMOTE_APP_CONTENT_REGEX}" | sort| uniq )
echo "${SORTED_REMOTE_APP_CONTENT_REGEX}" > "${REMOTE_APP_CONTENT_REGEX}" echo "${SORTED_REMOTE_APP_CONTENT_REGEX}" > "${REMOTE_APP_CONTENT_REGEX}"
DIFF=$( diff "${TMP_CURRENT_APP_CONTENT_FILE}" "${REMOTE_APP_CONTENT_REGEX}" |sed -e "1,3d;" ) DIFF=$( diff "${TMP_CURRENT_APP_CONTENT_FILE}" "${REMOTE_APP_CONTENT_REGEX}" |sed -e "1,3d;" )
@ -191,6 +220,9 @@ sed -i -e 's@\\@\\\\\\@g' "${TMP_CLISH_TRANSACTION_FILE}"
if [ "${DRY_RUN}" -eq "0" ];then if [ "${DRY_RUN}" -eq "0" ];then
clish -f "${TMP_CLISH_TRANSACTION_FILE}" clish -f "${TMP_CLISH_TRANSACTION_FILE}"
echo "$? exit code from clish -f"
else
echo "Running in DRY-RUN MODE"
fi fi
echo "Finished Transaction" echo "Finished Transaction"
@ -199,8 +231,8 @@ echo "Cleaning up files ..."
if [ "${CLEANUP_AFTER}" -eq "1" ];then if [ "${CLEANUP_AFTER}" -eq "1" ];then
rm -v "${TMP_DOWNLOAD_FILE}" rm -v "${TMP_DOWNLOAD_FILE}"
rm -v "${TMP_CLISH_UPDATE_FILE}" rm -v "${TMP_CLISH_UPDATE_FILE}"
rm -v "${TMP_CURRENT_APP_CONTENT_FILE}" rm -v "${TMP_CURRENT_APP_CONTENT_FILE}"
rm -v "${REMOTE_APP_CONTENT_REGEX}" rm -v "${REMOTE_APP_CONTENT_REGEX}"
rm -v "${TMP_CURRENT_CONFIG_FILE}" rm -v "${TMP_CURRENT_CONFIG_FILE}"
rm -v "${TMP_DIFF_FILE}" rm -v "${TMP_DIFF_FILE}"
rm -v "${TMP_CLISH_TRANSACTION_FILE}" rm -v "${TMP_CLISH_TRANSACTION_FILE}"
@ -209,8 +241,8 @@ else
echo "Don't forget to cleanup the files:" echo "Don't forget to cleanup the files:"
echo "${TMP_DOWNLOAD_FILE}" echo "${TMP_DOWNLOAD_FILE}"
echo "${TMP_CLISH_UPDATE_FILE}" echo "${TMP_CLISH_UPDATE_FILE}"
echo "${TMP_CURRENT_APP_CONTENT_FILE}" echo "${TMP_CURRENT_APP_CONTENT_FILE}"
echo "${REMOTE_APP_CONTENT_REGEX}" echo "${REMOTE_APP_CONTENT_REGEX}"
echo "${TMP_CURRENT_CONFIG_FILE}" echo "${TMP_CURRENT_CONFIG_FILE}"
echo "${TMP_DIFF_FILE}" echo "${TMP_DIFF_FILE}"
echo "${TMP_CLISH_TRANSACTION_FILE}" echo "${TMP_CLISH_TRANSACTION_FILE}"