python调用mrjob实现hadoop的mapreduce日志解析

水莹儿

[root@kspc ~]# python mo.py -r local  <10.7.17.7-dnsquery.log.1> output　　
no configs found; falling back on auto-configuration
　　
no configs found; falling back on auto-configuration
　　
creating tmp directory /tmp/mo.root.20131224.040935.241241
　　
reading from STDIN
　　
writing to /tmp/mo.root.20131224.040935.241241/step-0-mapper_part-00000
　　
> /usr/bin/python mo.py --step-num=0 --mapper /tmp/mo.root.20131224.040935.241241/input_part-00000 | sort | /usr/bin/python mo.py --step-num=0 --combiner > /tmp/mo.root.20131224.040935.241241/step-0-mapper_part-00000
　　
writing to /tmp/mo.root.20131224.040935.241241/step-0-mapper_part-00001
　　
> /usr/bin/python mo.py --step-num=0 --mapper /tmp/mo.root.20131224.040935.241241/input_part-00001 | sort | /usr/bin/python mo.py --step-num=0 --combiner > /tmp/mo.root.20131224.040935.241241/step-0-mapper_part-00001
　　
Counters from step 1:
　　
  (no counters found)
　　
writing to /tmp/mo.root.20131224.040935.241241/step-0-mapper-sorted
　　
> sort /tmp/mo.root.20131224.040935.241241/step-0-mapper_part-00000 /tmp/mo.root.20131224.040935.241241/step-0-mapper_part-00001
　　
writing to /tmp/mo.root.20131224.040935.241241/step-0-reducer_part-00000
　　
> /usr/bin/python mo.py --step-num=0 --reducer /tmp/mo.root.20131224.040935.241241/input_part-00000 > /tmp/mo.root.20131224.040935.241241/step-0-reducer_part-00000
　　
writing to /tmp/mo.root.20131224.040935.241241/step-0-reducer_part-00001
　　
> /usr/bin/python mo.py --step-num=0 --reducer /tmp/mo.root.20131224.040935.241241/input_part-00001 > /tmp/mo.root.20131224.040935.241241/step-0-reducer_part-00001
　　
Counters from step 1:
　　
  (no counters found)
　　
Moving /tmp/mo.root.20131224.040935.241241/step-0-reducer_part-00000 -> /tmp/mo.root.20131224.040935.241241/output/part-00000
　　
Moving /tmp/mo.root.20131224.040935.241241/step-0-reducer_part-00001 -> /tmp/mo.root.20131224.040935.241241/output/part-00001
　　
Streaming final output from /tmp/mo.root.20131224.040935.241241/output
　　
removing tmp directory /tmp/mo.root.20131224.040935.241241