Category Archives: Scripting

Linux resource checks notes

top

1 -> CUP utilisation
> -> order by memory

check for CPU Performance
Optimized waiting%
(press '1' to see all the CPUs)

-----------------

I/O - check for %util
iostat -kx 1 1000

-----------------

Memory
free -m
free -m | grep "buffers/cache" | awk '{print $3}'


atop utility

 

 

Rsync – exclude

>> Exclude .txt files [! CASE SENSITIVE]
$ rsync -avz --exclude '*.txt' source/ destination/

>> Exclude from file list
$cat exclude-list.txt 
*.JPG
*.TMP
*.PDF
*.jpg
*.tmp
*.pdf
*.zip
relative/path1/
relative/path2/

$ rsync -avz --exclude-from 'exclude-list.txt' /source/path/ /dest/path/ | tee rsync-report.txt


>> Exclude directory 
$ rsync -avz --exclude 'folder1_within_source' --exclude 'folder2_within_source/subfolder2' source/ destination/

 

recap utility

# Install recap
git clone https://github.com/rackerlabs/recap.git
cd recap
./recap-installer 


# If there is no MySQL or it can;t connect to it edit:
/etc/recap
                                USEMYSQL=no
                                USEMYSQLPROCESSLIST=no

Edit cron if you want to run often than 10 minutes:
                                /etc/cron.d/recap 


=================================================

LOG ANALYSIS SCRIPTS

>> Check for memory usage
grep "buffers/cache" /var/log/recap/resources_* | sed 's/^.*resources_\(.*\)\.log.*cache:\s*\([0-9]*\)\s*\([0-9]*\)$/FREE: \3| USED: \2| DATE: \1/ '

>> Running the following to check if the memory went less than 4999MB
grep "buffers/cache" /var/log/recap/resources_* | sed 's/^.*resources_\(.*\)\.log.*cache:\s*\([0-9]*\)\s*\([0-9]*\)$/FREE: \3| USED: \2| DATE: \1/ ' | egrep -v "FREE: [65][0-9]{3}\|" | wc -l

 

Bash log redirect / stdout and stderr

# For cron/crontab
*/5 * * * * /path/myscript.sh > /dev/null 2>&1

==================================================
# Redirect ALL output/error automatically to a file 
# AND print to console too
LOG_OUTPUT=output_error.log

exec 1> >(tee -i ${LOG_OUTPUT}) 2>&1

==================================================
# *ALL* redirected to the log files: NO console output at all
LOG_OUTPUT=output.log

exec 1>>${LOG_OUTPUT} 2>&1

==================================================
# All redirected to 2 different log files
LOG_OUTPUT=etup_output.log
LOG_ERROR=setup_error.log

exec 3>&1 1>>${LOG_OUTPUT}
exec 2>>${LOG_ERROR}

# use 'P "my message"' instead of echo
P () {
# Print on console AND file
echo -e "\n$1" | tee /dev/fd/3

# Print ONLY on console
#echo -e "\n$1" 1>&3
}

==================================================
# ALL stdout and stderr to $LOG_OUTPUT
# Also stderr to $LOG_ERROR (for extra checks)
# P function to print to the console AND logged into $LOG_OUTPUT
LOG_OUTPUT=output.log
LOG_ERROR=error.log

exec 3>&1 1>>${LOG_OUTPUT}
exec 2> >(tee -i ${LOG_ERROR}) >> ${LOG_OUTPUT}

# use 'P "my message"' instead of echo
P () {
# Print on console AND file
echo -e "$1" | tee /dev/fd/3

# Print ONLY on console
#echo -e "\n$1" 1>&3
}

# use 'P "my message"' instead of echo to print in BLUE
P () {
BLUE='\033[1;34m'
NC='\033[0m' # No Color
echo -e "\n${BLUE}${1}${NC}" | tee /dev/fd/3
}