Differences

This shows you the differences between two versions of the page.

@@ Line 1: / Line 1: @@
+====== [SCRIPT] Purge script ======
+====== Description ======
+Have you ever fight with logrotate configuration files? Did you survived? Cool, this document is not for you xD
+This is script is a very simple method to drop unnecessary files from your filesystems (logs?) and rotate them so they don't growth forever :-P
+This script write 2 logs each time :
+  * ''$(dirname purge_files.sh)/logs/purge_files_$(date +%Y%m%d%H%M).log''
+  * ''$(dirname purge_files.sh)/logs/purge_files_$(date +%Y%m%d%H%M).err''
+So you will have detailed the script actions+errors there
+====== Instructions ======
+===== Usage =====
+<code bash>/home/scripts/purge_files/purge_files.sh</code>
+===== Configuration =====
+It's a file-based configuration, the variable inside the script which sets the file is:
+<code>
+CONFIGFILE=/home/scripts/purge_files/purge_files.config
+</code>
+==== Config file sytax ====
+Each line in the config file is set of file to be analysed and purged/rotated, the syntax is:
+<code>#path,mask,action,param1,param2,param3</code>
+Where:
+  * Path is the folder from which the script begin searching (serching with find)
+  * Mask: file mask for matching files, the script uses ''find -name'' (Ex:  *gz, *log, *txt)
+  * Action: Any of the avalaible actions detailed here down.
+Y las opciones disponibles son (explicadas como parámetros de find)
+=== Actions ===
+^ path ^ mask ^ action ^ param1 ^ param2 ^ param3 ^
+| /path | ? | bydate | mtime | maxdepth | N/A |
+| /path | ? | bymin | mmin | maxdepth | N/A |
+| /path | ? | rotate | size | maxdepth | N/A |
+| /path | ? | rotatezip | size | maxdepth | gzip parameters |
+| /path | ? | zip | mtime | maxdepth | gzip parameters |
+| /path | ? | dirbydate | mtime | maxdepth | N/A |
+| /path | ? | dirbymin | mmin | maxdepth | N/A |
+Description:
+  * bydate : Delete files based on modification time (''find -mtime +X -type f'')
+  * bymin : Drop files based on modification minute (''find -mmin +X -type f'')
+  * dirbydate : Drop folders based on modification time. (''find -mtime +X -type d'')
+  * dirbymin : Drop folders based on modification minute (''find -mmin +X -type d'')
+  * rotate : Rotate the file (''cat file > file.date && > file'')
+  * rotatezip : Rorate and compress the file (''gzip -c file > file.date.gz && > file'')
+  * zip : Compress the file (''gzip file'')
+==== Sample Config ====
+<code>#path,mask,action,param1,param2,param3
+# Allways purge the script logs
+/home/scripts/purge_files/logs,*.log,bydate,+4,
+/home/scripts/purge_files/logs,*.err,bydate,+4,
+# Target: nginx logs
+/var/log/nginx,*.gz,bydate,+20,,
+# Objetivo : Oracle listerner trace files
+/u01/app/oracle/diag/tnslsnr,*trc,bydate,+15,4,
+/u01/app/oracle/diag/tnslsnr,*trm,bydate,+15,4,
+</code>
+===== Script Code =====
+<file bash purge_files.sh>
+#!/bin/bash
+#
+# (C) dodger@ciberterminal.net
+#
+# Exit codes:
+#  1 : No config file
+#  2 : Wrong option in the config file
+#  3 :
+#  4 :
+#  5 :
+#  6 :
+########################################################################
+#
+# CONSTANTS
+#
+########################################################################
+# colors
+LIGHTGREEN="\033[1;32m"
+LIGHTRED="\033[1;31m"
+WHITE="\033[0;37m"
+RESET="\033[0;00m"
+########################################################################
+#
+# / CONSTANTS
+#
+########################################################################
+########################################################################
+#
+# VARIABLES
+#
+########################################################################
+MYDATE=$(date +%Y%m%d%H%M)
+CONFIGFILE=/home/scripts/purge_files/purge_files.config
+LOGDIR=$(dirname $0)/logs/
+########################################################################
+#
+# / VARIABLES
+#
+########################################################################
+########################################################################
+#
+# FUNCTIONS
+#
+########################################################################
+usage()
+{
+    printf "%s${LIGHTRED}USAGE:${RESET}
+    $0
+    PLEASE READ
+    https://sites.google.com/a/ciberterminal.net/sistemas/oracle/documentacion-tecnica/rotado-de-logs-y-trazas\n"
+    # VERY INITIAL CHECKS
+}
+remove_files()
+{
+    local AUX=""
+    local OPT="-${1}"
+    local DIR="${2}"
+    if [[ "${DIR,,}" = "dir" ]] ; then
+        local TYPEOF="-type d"
+        local FINDCMD="-print | xargs rm -fr"
+        local MESSAGE="$(date +%Y%m%d%H%M%S) : Deleting folders inside ${FOLDER}, with filemask ${MASK}, ${OPT} ${PARAM1}"
+    else
+        local TYPEOF="-type f"
+        local FINDCMD="-delete"
+        local MESSAGE="$(date +%Y%m%d%H%M%S) : Deleting files on ${FOLDER}, with filemask ${MASK}, ${OPT} ${PARAM1}"
+    fi
+    [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && MESSAGE="%s${MESSAGE} and maxdepth ${PARAM2}"
+    printf "%s${MESSAGE}\n"
+    eval find ${FOLDER} ${AUX} -name "${MASK}" ${TYPEOF} ${OPT} ${PARAM1} ${FINDCMD}
+}
+rotate_files()
+{
+    local USEZIP="$1"
+    local MESSAGE="$(date +%Y%m%d%H%M%S) : Rotating files on ${FOLDER}, with filemask ${MASK}, -mtime ${PARAM1}"
+    [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && MESSAGE="${MESSAGE}, maxdepth ${PARAM2}"
+    printf "%s${MESSAGE}\n"
+#     printf "%s$(date +%Y%m%d%H%M%S) : Rotating files on ${FOLDER}, with filemask ${MASK}, -mtime ${PARAM1}"
+#     [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && printf "%s, maxdepth ${PARAM2}"
+    case $USEZIP in
+        "zip" )
+            [[ ! "${PARAM3}" ]] && PARAM3="-9fv"
+            local MOVECMD="gzip -c ${PARAM3}"
+            local DSTFILE="${MYDATE}.gz"
+            printf "%s and gzipping with ${PARAM3}\n"
+        ;;
+        "nozip" )
+            local MOVECMD="cat"
+            local DSTFILE="${MYDATE}"
+            printf "%s\n"
+        ;;
+        * )
+            printf "%s${LIGHTRED}ERROR ON rotate_files function${RESET}"
+            return 1
+        ;;
+    esac
+    while read FILE ; do
+        echo ${FILE}
+        ${MOVECMD} "${FILE}" > "${FILE}.${DSTFILE}"
+         > ${FILE}
+    done < <(find ${FOLDER} ${AUX} -name "${MASK}" -type f -size ${PARAM1^^})
+}
+########################################################################
+#
+# / FUNCTIONS
+#
+########################################################################
+########################################################################
+#
+#  MAIN
+#
+########################################################################
+[ ! -d ${LOGDIR} ] && mkdir -p ${LOGDIR}
+exec 1>> ${LOGDIR}/$(basename $0 .sh)_${MYDATE}.log
+exec 2>> ${LOGDIR}/$(basename $0 .sh)_${MYDATE}.err
+if [ ! -f ${CONFIGFILE} ]; then
+        echo "No configuration file found"
+        exit 1
+fi
+# workaround for eval "bug"
+AUXDIR="/tmp/$(date +%s)"
+mkdir ${AUXDIR}
+cd ${AUXDIR}
+#/ workaround for eval "bug"
+while read LINE ; do
+    if [[ ! "${LINE,,}" =~ ^\/[a-z0-9]{1,}\/[\.\/\_\ a-z0-9\-]{1,},[\*\.\_\ \#a-z0-9\-]{3,20},(dirbydate|dirbymin|bydate|bymin|rotate|rotatezip|zip),\+[0-9]{1,3}(|[kmg]),(|[0-9]{1,3}),(|\-[a-z0-9]{1,10})$ ]] ; then
+        printf "%sskipping ${LINE}\nnot matching the config pattern, read instructions\n"
+        continue
+    fi
+    FOLDER=$(echo $LINE | cut -d',' -f1)
+    MASK=$(echo $LINE | cut -d',' -f2)
+    ACTION=$(echo $LINE | cut -d',' -f3)
+    PARAM1=$(echo $LINE | cut -d',' -f4)
+    PARAM2=$(echo $LINE | cut -d',' -f5)
+    PARAM3=$(echo $LINE | cut -d',' -f6)
+    case ${ACTION,,} in
+        "bydate" )
+            remove_files mtime
+        ;;
+        "bymin" )
+            remove_files mmin
+        ;;
+       "dirbydate" )
+            remove_files mtime dir
+        ;;
+        "dirbymin" )
+            remove_files mmin dir
+        ;;
+        "rotate" )
+            rotate_files nozip
+        ;;
+        "rotatezip" )
+            rotate_files zip
+        ;;
+        "zip" )
+            printf "%s$(date +%Y%m%d%H%M%S) : Gzipping files on ${FOLDER}, with filemask ${MASK}, ${OPT} ${PARAM1}"
+            [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && printf "%s and maxdepth ${PARAM2}\n" || printf "%s\n"
+            [[ ! "${PARAM3}" ]] && PARAM3="-9fv"
+            find ${FOLDER} ${AUX} -name "${MASK}" -type f -mtime ${PARAM1} -exec gzip ${PARAM3} {} \;
+        ;;
+        * )
+            usage
+            exit 2
+        ;;
+    esac
+done < <(cat ${CONFIGFILE} | egrep -v "^#|^$")
+# workaround for eval "bug"
+cd ${OLDPWD}
+rm -fr ${AUXDIR}
+#/ workaround for eval "bug"
+########################################################################
+#
+# / MAIN
+#
+########################################################################
+</file>