shell之日志处理
#!/bin/bash#by lc at 20160428 for hadoop(grep_recharge)
DATE=`date +%F_%H\:%M\:%S`
YTDATE=`date -d "-1 day" +%Y%m%d`
RESTDIR=/home/hadoop/prodata/rest
DATEDIR=/home/hadoop/prodata/rest/$YTDATE
RESTFILE=${DATEDIR}/recharge-$YTDATE.log
#RESTFILE=recharge-20160427.log
#NEWFILE=parse-recharge.log
NEWFILE=${DATEDIR}/parse-recharge-$YTDATE.log
LOGFILE=/server/scripts/grep_recharge.log
function CheckFile(){
> $NEWFILE
WCRESTFILE=`wc -l $RESTFILE`
echo '########################' >> $LOGFILE
echo "${DATE} GO..." >> $LOGFILE
echo "$WCRESTFILE" >> $LOGFILE
if [ ! -e $RESTFILE ]
then
echo "$RESTFILE CANT FIND" >>$LOGFIL
exit 1
fi
}
function GrepFile(){
egrep -v "rechMoney|money_order" ${RESTFILE}|grep -v 'cardNum' |awk -F'[{]' '{print $1}'|sed -nr "s#(.*)#\1\{\}#gp" >${NEWFILE}
#######CardNum
egrep -v "rechMoney|money_order" ${RESTFILE}|grep 'cardNum'|grep "\{8\}\"\}$"|sed -nr 's#(.*)\{.*\}#\1\{\}#gp' >>${NEWFILE}
egrep -v "rechMoney|money_order" ${RESTFILE}|grep 'cardNum'|grep -v "\{8\}\"\}$"|tr '{}' '1'|sed -nr 's#(.*)#\1\|\{\}#gp' >>${NEWFILE}
######Money_Order
grep "money_order" ${RESTFILE}|sed 's#\\##g'|tr '{' '#'|sed -nr 's#(.*)\#.*\#.*money_order(.*)#\1\{\"rechMoney\2#gp'|sed 's#\###g'|sed 's#iPhone,#iPhone#g'|sed 's#iPod,#iPod#g'|sed 's#iPad,#iPad#g'|awk -F[,] '{print $1"\,\"recharge\"\:\"param\"\}"}'|sed 's#\"data\":##g' >>${NEWFILE}
#######rechMoney
grep "rechMoney" ${RESTFILE}|sed 's#\\##g'|tr '{' '#'|sed -nr "s#(.*)\#.*rechMoney(.*)#\1\"rechMoney\2#gp"|sed 's#\###g'|sed 's#iPhone,#iPhone#g'|sed 's#iPod,#iPod#g'|sed 's#iPad,#iPad#g'|awk -F'[,]' '{print $1}'|sed 's#\}##g'|awk -F'["]' '{print $1"\{\""$2"\""$3"\""$4"\"\,\"recharge\"\:\"param\"\}"}' >>${NEWFILE}
echo "`wc -l ${NEWFILE}` " >> $LOGFILE
if [ $? -ne 0 ]
then
echo "`date +%F_%H\:%M\:%S`NOT OK" >> $LOGFILE
else
echo "`date +%F_%H\:%M\:%S` OK" >>$LOGFILE
fi
}
main (){
CheckFile
GrepFile
}
main
页:
[1]