利用flume-ng进行日志收集

3r211

一、安装环境
agent:192.168.7.101
hdfs:192.168.7.70(namenode)
     192.168.7.71(datanode)
     192.168.7.72(datanode)
     192.168.7.73(datanode)
操作系统：CentOS 6.3 x86_64
须用到的软件包:    jdk-1.7.0_65-fcs.x86_64 flume-ng-1.5.0 flume-ng-agent-1.5.0 hadoop-2.3.0+cdh5.1.0

cat /etc/hosts
192.168.7.70 cdh1
192.168.7.71 cdh2
192.168.7.72 cdh3
192.168.7.73 cdh4

二、配置flume-ng-agent
1、涉及到的日志文件:
/home/logs/bizservice/bizservice.log
/home/logs/base/base.log
/home/logs/agent/agent.log
/home/logs/thirdser/thirdser.log

2、安装flume-ng-agent
yum -y install flume-ng* jdk-1.7.0_65-fcs.x86_64

3、配置/etc/profile.d/java.sh并生效
echo 'JAVA_HOME=/usr/java/latest' >> /etc/profile.d/java.sh
echo 'PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile.d/java.sh
echo 'export JAVA_HOME PATH' >> /etc/profile.d/java.sh
source /etc/profile.d/java.sh

4、利用template.conf、flume.sh生成/etc/flume-ng/conf/flume.conf
cat /tmp/template.conf

# Name the  components on this agent
aa.sources =  rr
aa.sinks =  kk
aa.channels  = cc

#  Describe/configure the source
aa.sources.rr.type  = exec
aa.sources.rr.command  = tail -F AGENT1
aa.sources.rr.channels  = cc
aa.sources.rr.bind  = 0.0.0.0
aa.sources.rr.port  = 4141


# Describe  the sink
aa.sinks.kk.type  = hdfs
aa.sinks.kk.channel  = cc
aa.sinks.kk.hdfs.path  =  hdfs://cdh1:8020/flume/AGENT2/%y-%m-%d/%H%M/%S
aa.sinks.kk.hdfs.filePrefix  = AGENT2%{host}
aa.sinks.kk.hdfs.round  = true
aa.sinks.kk.hdfs.roundValue  = 10
aa.sinks.kk.hdfs.roundUnit  = minute
aa.sinks.kk.hdfs.useLocalTimeStamp = true  

# Use a  channel which buffers events in memory
aa.channels.cc.type  = memory
aa.channels.cc.capacity  = 1000
aa.channels.cc.transactionCapacity  = 100
# Bind the  source and sink to the channel
aa.sources.rr.channels  = cc
aa.sinks.kk.channel  = cc

cat flume.sh
#!/bin/bash
source /etc/profile;
cd /etc/flume-ng/conf/
OBJECT=`find /home/logs/ -name *.log -type f |egrep "bizservice.log|base.log|agent.log|thirdser.log"|awk -F '/' '{print $NF}'|sed 's/.log//g'`
find /home/logs/ -name *.log -type f |egrep "bizservice.log|base.log|agent.log|thirdser.log"|tr ' ' '\n' > /tmp/TEMP
rm -f flume.conf
for I in $OBJECT
do
    \cp -rf /tmp/template.conf "$I".conf
    PATHE=`grep "$I" /tmp/TEMP`
    sed -i 's/aa/'$I'_a1/g' "$I".conf
    sed -i 's/rr/'$I'_r1/g' "$I".conf
    sed -i 's/kk/'$I'_k1/g' "$I".conf
    sed -i 's/cc/'$I'_c1/g' "$I".conf
    sed -i 's#AGENT1#'$PATHE'#g' "$I".conf
    sed -i 's/AGENT2/'$I'/g' "$I".conf
    cat  "$I".conf >> flume.conf
    rm -f "$I".conf
done
rm -f TEMP

5、修改/etc/init.d/flume-ng-agent
DEFAULT_FLUME_AGENT_NAME=`cat /etc/flume-ng/conf/flume.conf|grep a1|awk -F . '{print $1}'|sort -u|tr '\n' ' '`

        for FLUME_AGENT in $FLUME_AGENT_NAME
        do
  /bin/su -s /bin/bash -c "/bin/bash -c 'echo \$\$ >${FLUME_PID_FILE} && exec ${EXEC_PATH} agent --conf $FLUME_CONF_DIR --conf-file $FLUME_CONF_FILE --name $FLUME_AGENT >>${FLUME_LOG_DIR}/flume.init.log 2>&1' &" $FLUME_USER
        done

6、启动flume-ng-agent
/etc/init.d/flume-ng-agent start

7、查看日志/var/log/flume-ng/flume.log，显示已写入数据到hdfs中。

03 九月 2014 14:09:39,445 INFO  [hdfs-bizservice_k1-call-runner-9] (org.apache.flume.sink.hdfs.BucketWriter$8.call:673)  - Renaming hdfs://cdh1:8020/flume/bizservice/14-09-03/1400/00/bizservice.1409724409419.tmp to hdfs://cdh1:8020/flume/bizservice/14-09-03/1400/00/bizservice.1409724409419
03 九月 2014 14:09:39,470 INFO  [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.sink.hdfs.BucketWriter.open:261)  - Creating hdfs://cdh1:8020/flume/bizservice/14-09-03/1400/00/bizservice.1409724409420.tmp