python 网页抓取经验

a2005147

一定要设置好user-agnet 和accept

#coding:utf-8
'''
Created on 2009-7-15
@author: Administrator
'''
import urllib2
import newhttplib
import lxml.html as x  
def getmusic(num,soc=None):
s=num.split(':')
s[0]=s[0].strip()
s[1]=s[1].strip()
s[2]=s[2].strip()
h1 = newhttplib.HTTPConnection('10.0.0.172',80)
hs={'Accept': 'text/html,application/xhtml+xml,application/xml','User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1) Gecko/20090624 Firefox/3.5'}
h1.auto_open = 0
h1.connect(soc)
url='http://218.200.160.29/s3/i/app/search/musicResult.jsp?qd=1956&CH=12530-wap-lslb&v=1864&tplpath=/s3/i/qrc/&type=all&keyword='+urllib2.quote(s[0].encode('utf-8')+" "+s[1].encode('utf-8'))
h1.request("GET",url,headers=hs)
r1 = h1.getresponse()
print r1.getheader('Content-Type')
content=r1.read()   
#content=content.decode('utf-8')
doc=x.document_fromstring(content)  
alist=doc.xpath("//a")
print s[0],s[1],len(alist)
for a in alist:
c1= a.text_content()
try:
if c1.find(s[0])>=0 and c1.find(s[1])>=0:
print c1
print c1.find(s[0]),c1.find(s[1])
nexturl='http://218.200.160.29'+a.get('href')
print nexturl
h1.request("GET",nexturl,headers=hs)
r1 = h1.getresponse()
content=r1.read()
print r1.getheaders()
#content=content.decode('utf-8')
print content
f=open('./b.html','wb')
f.write(content)
f.write(nexturl)
f.close()
doc=x.document_fromstring(content)  
blist=doc.xpath("//a")
print 'blen=',len(blist)
for b in blist:
bcontent=b.text_content()
if bcontent.find(u'高潮版')>=0:
print bcontent
bhref= b.get('href')
nnexturl='http://218.200.160.29'+bhref
h1.request("GET",nnexturl,headers=hs)
r1 = h1.getresponse()
ccontent=r1.read()
#content=content.decode('utf-8')
#print ccontent
doc=x.document_fromstring(ccontent)  
clist=doc.xpath("//a")
print 'clen=',len(clist)
for c in clist:
chref=c.get('href')
if chref.find('218.200.160.10')>=0:
print chref
sb2=s[2][11:]
print sb2
if chref.find(sb2)>=0:
print u'找到匹配歌曲开始下载。。。'
h1.request("GET",chref)
r1 = h1.getresponse()
print r1.status
print r1.getheaders()
except BaseException,e:
print e
if __name__ == '__main__':
#f=open('./test1.htm','wb')
#con=f.read()  
#print con
getmusic('相信:曾建军:600902000005714466')