Summary: rockuse more memory that asked to. Monitor and report. Test Plan: run the pro with conditions to simulate the overusage. It should report that the process is using more memory than needed. Reviewers: yhchiang, rven, sdong, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D33249main
Ramki Balasubramanian
10 years ago
parent
5f00af4570
commit
5d1151deba
@ -0,0 +1,102 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# |
||||||
|
#(c) 2004-present, Facebook Inc. All rights reserved. |
||||||
|
# |
||||||
|
#see LICENSE file for more information on use/redistribution rights. |
||||||
|
# |
||||||
|
|
||||||
|
# |
||||||
|
#dbench_monitor: monitor db_bench process for violation of memory utilization |
||||||
|
# |
||||||
|
#default usage will monitor 'virtual memory size'. See below for standard options |
||||||
|
#passed to db_bench during this test. |
||||||
|
# |
||||||
|
# See also: ./pflag for the actual monitoring script that does the work |
||||||
|
# |
||||||
|
#NOTE: |
||||||
|
# You may end up with some /tmp/ files if db_bench OR |
||||||
|
# this script OR ./pflag was killed unceremoniously |
||||||
|
# |
||||||
|
# If you see the script taking a long time, trying "kill" |
||||||
|
# will usually cleanly exit. |
||||||
|
# |
||||||
|
# |
||||||
|
DIR=`dirname $0` |
||||||
|
LOG=/tmp/`basename $0`.$$ |
||||||
|
DB_BENCH="$DIR/../db_bench"; |
||||||
|
PFLAG=${DIR}/pflag |
||||||
|
|
||||||
|
usage() { |
||||||
|
cat <<HELP; exit |
||||||
|
|
||||||
|
Usage: $0 [-h] |
||||||
|
|
||||||
|
-h: prints this help message |
||||||
|
|
||||||
|
This program will run the db_bench script to monitor memory usage |
||||||
|
using the 'pflag' program. It launches db_bench with default settings |
||||||
|
for certain arguments. You can change the defaults passed to |
||||||
|
'db_bench' program, by setting the following environment |
||||||
|
variables: |
||||||
|
|
||||||
|
bs [block_size] |
||||||
|
ztype [compression_type] |
||||||
|
benches [benchmarks] |
||||||
|
reads [reads] |
||||||
|
threads [threads] |
||||||
|
cs [cache_size] |
||||||
|
vsize [value_size] |
||||||
|
comp [compression_ratio] |
||||||
|
num [num] |
||||||
|
|
||||||
|
See the code for more info |
||||||
|
|
||||||
|
HELP |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
[ ! -x ${DB_BENCH} ] && echo "WARNING: ${DB_BENCH} doesn't exist, abort!" && exit -1; |
||||||
|
|
||||||
|
[ "x$1" = "x-h" ] && usage; |
||||||
|
|
||||||
|
trap 'rm -f ${LOG}; kill ${PID}; echo "Interrupted, exiting";' 1 2 3 15 |
||||||
|
|
||||||
|
touch $LOG; |
||||||
|
|
||||||
|
: ${bs:=16384} |
||||||
|
: ${ztype:=zlib} |
||||||
|
: ${benches:=readwhilewriting} |
||||||
|
: ${reads:=$((1*1024*1024))}; |
||||||
|
: ${threads:=8} |
||||||
|
: ${vsize:=2000} |
||||||
|
: ${comp:=0.5} |
||||||
|
: ${num:=10000} |
||||||
|
: ${cs:=$((1*1024*1024*1024))}; |
||||||
|
|
||||||
|
DEBUG=1 #Set to 0 to remove chattiness |
||||||
|
|
||||||
|
|
||||||
|
if [ "x$DEBUG" != "x" ]; then |
||||||
|
# |
||||||
|
#NOTE: under some circumstances, --use_existing_db may leave LOCK files under ${TMPDIR}/rocksdb/* |
||||||
|
#cleanup the dir and re-run |
||||||
|
# |
||||||
|
echo DEBUG: Will run $DB_BENCH --block_size=$bs --compression_type=$ztype --benchmarks="$benches" --reads="$reads" --threads="$threads" --cache_size=$cs --value_size=$vsize --compression_ratio=$comp --num=$num --use_existing_db |
||||||
|
|
||||||
|
fi |
||||||
|
|
||||||
|
$DB_BENCH --block_size=$bs --compression_type=$ztype --benchmarks="$benches" --reads="$reads" --threads="$threads" --cache_size=$cs --value_size=$vsize --compression_ratio=$comp --num=$num --use_existing_db >$LOG 2>&1 & |
||||||
|
|
||||||
|
if [ $? -ne 0 ]; then |
||||||
|
warn "WARNING: ${DB_BENCH} did not launch successfully! Abort!"; |
||||||
|
exit; |
||||||
|
fi |
||||||
|
PID=$! |
||||||
|
|
||||||
|
# |
||||||
|
#Start the monitoring. Default is "vsz" monitoring for upto cache_size ($cs) value of virtual mem |
||||||
|
#You could also monitor RSS and CPUTIME (bsdtime). Try 'pflag -h' for how to do this |
||||||
|
# |
||||||
|
${PFLAG} -p $PID -v |
||||||
|
|
||||||
|
rm -f $LOG; |
||||||
@ -0,0 +1,217 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# |
||||||
|
#(c) 2004-present, Facebook, all rights reserved. |
||||||
|
# See the LICENSE file for usage and distribution rights. |
||||||
|
# |
||||||
|
|
||||||
|
trap 'echo "Caught exception, dying"; exit' 1 2 3 15 |
||||||
|
|
||||||
|
ME=`basename $0` |
||||||
|
SERVER=`hostname` |
||||||
|
|
||||||
|
#parameters used |
||||||
|
# |
||||||
|
Dump_Config=0 |
||||||
|
DEBUG= |
||||||
|
OS=`/bin/uname -s` |
||||||
|
VMEM= |
||||||
|
RSS= |
||||||
|
CPU= |
||||||
|
VERBOSE= |
||||||
|
VAR= |
||||||
|
LIMIT= |
||||||
|
ACTION= |
||||||
|
N= |
||||||
|
WAIT= |
||||||
|
|
||||||
|
# |
||||||
|
#supported OS: Linux only for now. Easy to add |
||||||
|
# |
||||||
|
oscheck() { |
||||||
|
case ${OS} in |
||||||
|
Linux) |
||||||
|
VMEM=vsz |
||||||
|
RSS=rss |
||||||
|
CPU=bsdtime |
||||||
|
;; |
||||||
|
*) |
||||||
|
die "Unsupported OS ${OS}. Send a bug report with OS you need supported. Thanks." |
||||||
|
;; |
||||||
|
esac |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
verbose() { |
||||||
|
if [ "x$DEBUG" != "x" ]; then |
||||||
|
echo "$@" >&2 |
||||||
|
fi |
||||||
|
} |
||||||
|
|
||||||
|
warn() { |
||||||
|
echo "$@" >&2 |
||||||
|
} |
||||||
|
|
||||||
|
die() { |
||||||
|
echo "ERROR: " "$@" >&2; |
||||||
|
exit; |
||||||
|
} |
||||||
|
|
||||||
|
dump_config() { |
||||||
|
cat <<EOCONFIG; |
||||||
|
$ME running on ${HOSTNAME} at `date` |
||||||
|
|
||||||
|
Configuration for this run: |
||||||
|
PID to monitor : ${PID} |
||||||
|
Resource monitored : ${VAR} |
||||||
|
Resource limit : ${LIMIT} |
||||||
|
Check every : ${WAIT} seconds |
||||||
|
No. of times run : ${N} |
||||||
|
What to do : ${ACTION} |
||||||
|
EOCONFIG |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
usage() { |
||||||
|
cat <<USAGE; exit |
||||||
|
$@ |
||||||
|
|
||||||
|
Usage ${ME} -p pid [-x {VMEM|RSS|CPU}] -l limit [-a {warn|die|kill}] [-n cycles] [-w wait] |
||||||
|
|
||||||
|
Monitor a process for set of violations. Options: |
||||||
|
|
||||||
|
-p: PID of process to monitor |
||||||
|
|
||||||
|
-x: metric to sense. Currently only VMEM/RSS/CPU are supported. Defaults to VMEM |
||||||
|
|
||||||
|
-l: what is the threshold/limit for the metric that is being sensed. |
||||||
|
Examples: "-l 100m", "-l 1.5g" (for VMEM/RSS), "-l 5:04" 5:04 in BSDTIME for CPU |
||||||
|
NOTE: defaults to 1GB |
||||||
|
|
||||||
|
-a: action. Currently {warn|die|kill} are supported. |
||||||
|
The default action is to 'warn'. Here is the behavior: |
||||||
|
|
||||||
|
warn: complain if usage exceeds threshold, but continue monitoring |
||||||
|
kill: complain, kill the db_bench process and exit |
||||||
|
die: if usage exceeds threshold, die immediately |
||||||
|
|
||||||
|
-n: number of cycles to monitor. Default is to monitor until PID no longer exists. |
||||||
|
|
||||||
|
-w: wait time per cycle of monitoring. Default is 5 seconds. |
||||||
|
|
||||||
|
-v: verbose messaging |
||||||
|
|
||||||
|
USAGE |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
#set default values if none given |
||||||
|
set_defaults_if_noopt_given() { |
||||||
|
|
||||||
|
: ${VAR:=vsz} |
||||||
|
: ${LIMIT:=1024000} |
||||||
|
: ${WAIT:=5} |
||||||
|
: ${N:=999999} |
||||||
|
: ${ACTION:=warn} |
||||||
|
} |
||||||
|
|
||||||
|
validate_options() { |
||||||
|
if [ "x$PID" = "x" -a $Dump_Config -ne 1 ]; then |
||||||
|
usage "PID is mandatory" |
||||||
|
fi |
||||||
|
} |
||||||
|
|
||||||
|
###### START |
||||||
|
|
||||||
|
|
||||||
|
while getopts ":p:x:l:a:n:t:vhd" opt; do |
||||||
|
case $opt in |
||||||
|
d) |
||||||
|
Dump_Config=1 |
||||||
|
;; |
||||||
|
h) |
||||||
|
usage; |
||||||
|
;; |
||||||
|
a) |
||||||
|
ACTION=${OPTARG}; |
||||||
|
;; |
||||||
|
v) |
||||||
|
DEBUG=1; |
||||||
|
;; |
||||||
|
p) |
||||||
|
PID=$OPTARG; |
||||||
|
;; |
||||||
|
x) |
||||||
|
VAR=$OPTARG; |
||||||
|
;; |
||||||
|
l) |
||||||
|
LIMIT=$OPTARG; |
||||||
|
;; |
||||||
|
w) |
||||||
|
WAIT=$OPTARG; |
||||||
|
;; |
||||||
|
n) |
||||||
|
N=$OPTARG; |
||||||
|
;; |
||||||
|
\?) |
||||||
|
usage; |
||||||
|
;; |
||||||
|
esac |
||||||
|
done |
||||||
|
|
||||||
|
oscheck; |
||||||
|
set_defaults_if_noopt_given; |
||||||
|
validate_options; |
||||||
|
|
||||||
|
if [ $Dump_Config -eq 1 ]; then |
||||||
|
dump_config; |
||||||
|
exit; |
||||||
|
fi |
||||||
|
|
||||||
|
Done=0 |
||||||
|
|
||||||
|
verbose "Trying ${N} times, Waiting ${WAIT} seconds each iteration"; |
||||||
|
|
||||||
|
while [ $Done -eq 0 ]; do |
||||||
|
VAL=`/bin/ps h -p $PID -o ${VAR} | perl -pe 'chomp; s/(.*)m/$1 * 1024/e; s/(.*)g/$1 * 1024 * 1024/e;'` |
||||||
|
if [ ${VAL:=0} -eq 0 ]; then |
||||||
|
warn "Process $PID ended without incident." |
||||||
|
Done=1; |
||||||
|
break; |
||||||
|
fi |
||||||
|
|
||||||
|
if [ $VAL -ge $LIMIT ]; then |
||||||
|
Done=1; |
||||||
|
else |
||||||
|
echo "Value of '${VAR}' (${VAL}) is less than ${LIMIT} for PID ${PID}" |
||||||
|
sleep $WAIT; |
||||||
|
fi |
||||||
|
if [ $Done -eq 1 ]; then |
||||||
|
|
||||||
|
if [ "$ACTION" = "kill" ]; then |
||||||
|
kill ${PID} || kill -3 ${PID} |
||||||
|
exit; |
||||||
|
|
||||||
|
elif [ "$ACTION" = "warn" ]; then |
||||||
|
|
||||||
|
# go back to monitoring. |
||||||
|
|
||||||
|
warn "`date` WARNING: ${VAR} breached threshold ${LIMIT}, actual is ${VAL}" |
||||||
|
Done=0 #go back to monitoring |
||||||
|
|
||||||
|
elif [ "$ACTION" = "die" ]; then |
||||||
|
warn "WARNING: dying without killing process ${PID} on ${SERVER}" |
||||||
|
warn "The process details are below: " |
||||||
|
warn "`ps -p ${PID} -o pid,ppid,bsdtime,rss,vsz,cmd,args`" |
||||||
|
warn "" |
||||||
|
|
||||||
|
#should we send email/notify someone? TODO... for now, bail. |
||||||
|
|
||||||
|
exit -1; |
||||||
|
|
||||||
|
fi |
||||||
|
else |
||||||
|
: |
||||||
|
#warn "INFO: PID $PID, $VAR = $VAL, limit ($LIMIT) not exceeded"; |
||||||
|
fi |
||||||
|
done |
||||||
|
|
||||||
Loading…
Reference in new issue