从nginx日志读取URL来做性能测试

mahonglin123456

　　做性能测试的时候都希望能模拟用户的行为，ningx记录了用户的行为，比如nginx一般会记录如下：
　　222.210.49.198 - 209sdj3lufa7sf32 -c2b3lp - [14/Sep/2012:22:59:02 +0800]"GET /search/?page=33&kw=%E6%B5%B7%E8%B4%BC&type=feed&_type= HTTP/1.1" 200 14466 "http://www.duitang.com/search/?kw=%E6%B5%B7%E8%B4%BC&type=feed" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER"
　　写了一个简单的python脚本从nginx读取URL来做性能测试：

# -*- coding: utf-8 -*-
import re
import urllib2
import json
import threading
import Queue
import os
host="http://7199.t.duitang.com"
thread_count = 1
running = True
def check(uri):
try:
url = "%s%s"%(host,uri)
content = urllib2.urlopen(url=url,timeout=2)
#print '%s %s'%(content.code,uri)
#print content.info()
#print content.msg
#print content.readlines()
#json.loads(content.readlines())
#print 'ok'
except Exception,e:
#print e
print '%s %s'%(e,uri)
#out = StringIO()
#print_exc(file=out)
#logger.info("check %s fail %s"%(server,out.getvalue()))
class Executor(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while running:
url = self.queue.get()
print queue.qsize()
check(url)
self.queue.task_done()
def analysis(readline,queue):
r = re.compile("\d+")
mc_url = re.compile("GET /[a-zA-Z/0-9_?=&]*")
result = {}
for read in readline:
ip=url=http_code=http_time = None
match =  mc_url.search(read)
if match:
url = str(match.group())[4:].strip()
queue.put(url)
def clean(a,b):
print 'exit'
running=False
exit(1)
from signal import signal,SIGTERM,SIGINT,SIGQUIT
signal(SIGTERM,clean)
signal(SIGINT,clean)
signal(SIGQUIT,clean)
if __name__ == "__main__":
queue = Queue.Queue()
for i  in range(thread_count):
t = Executor(queue)
t.setDaemon(True)
t.start()
f = open("by_category","r")
analysis(f.readlines(),queue)
print "total %s" %queue.qsize()
queue.join()
print 'exit'
running = False