python数据抓取分析（python + mongodb）
def parser(sn,url):try:
headers = {
。。。。。。
}
r = requests.get(url, headers=headers,timeout=30)
html = r.content
soup = BeautifulSoup(html,"lxml")
dt = {}
#partno
a = soup.find("meta",itemprop="mpn")
if a:
dt['partno'] = a['content']
#manufacturer
b = soup.find("meta",itemprop="manufacturer")
if b:
dt['manufacturer'] = b['content']
#description
c = soup.find("span",itemprop="description")
if c:
dt['description'] = c.get_text().strip()
#price
price = soup.find("table",class_="table table-condensed occalc_pa_table")
if price:
cost = {}
for i in price.find_all('tr'):
if len(i) > 1:
td = i.find_all('td')
key=td.get_text().strip().replace(',','')
val=td.get_text().replace(u'\u20ac','').strip()
if key and val:
cost = val
if cost:
dt['cost'] = cost
dt['currency'] = 'EUR'
#quantity
d = soup.find("input",id="ItemQuantity")
if d:
dt['quantity'] = d['value']
#specs
e = soup.find("div",class_="row parameter-container")
if e:
key1 = []
val1= []
for k in e.find_all('dt'):
key =k.get_text().strip().strip('.')
if key:
key1.append(key)
for i in e.find_all('dd'):
val =i.get_text().strip()
if val:
val1.append(val)
specs = dict(zip(key1,val1))
if specs:
dt['specs'] = specs
print dt
if dt:
db.update({'sn':sn},{'$set':dt})
print str(sn) +' insert successfully'
time.sleep(3)
else:
error(str(sn) + '\t' + url)
except Exception,e:
error(str(sn) + '\t' + url)
print "Don't data!"
页:
[1]