抓取百度贴吧

采集 网络爬虫吧 的所有贴吧信息

http://tieba.baidu.com/f?ie=utf-8&kw=%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB&fr=search

解决问题思路：

  • 确认需求数据在哪

    右键查看源代码

  • Fidder模拟发送数据

源码

  1. # -*- coding:utf-8 -*-
  2. import urllib2
  3. import urllib
  4. from lxml import etree
  5. import chardet
  6. import json
  7. import codecs
  10. def GetTimeByArticle(url):
  11.     request = urllib2.Request(url)
  12.     response = urllib2.urlopen(request)
  13.     resHtml = response.read()
  14.     html = etree.HTML(resHtml)
  15.     time = html.xpath('//span[@class="tail-info"]')[1].text
  16.     print time
  17.     return time
  20. def main():
  21.     output = codecs.open('tieba0812.json', 'w', encoding='utf-8')
  23.     for pn in range(0, 250, 50):
  25.         kw = u'网络爬虫'.encode('utf-8')
  27.         url = 'http://tieba.baidu.com/f?kw=' + urllib.quote(kw) + '&ie=utf-8&pn=' + str(pn)
  28.         print url
  29.         request = urllib2.Request(url)
  30.         response = urllib2.urlopen(request)
  32.         resHtml = response.read()
  33.         print resHtml
  35.         html_dom = etree.HTML(resHtml)
  36.         # print etree.tostring(html_dom)
  37.         html = html_dom
  39.         # site = html.xpath('//li[@data-field]')[0]
  40.         for site in html.xpath('//li[@data-field]'):
  41.             # print etree.tostring(site.xpath('.//a')[0])
  42.             title = site.xpath('.//a')[0].text
  43.             Article_url = site.xpath('.//a')[0].attrib['href']
  44.             reply_date = GetTimeByArticle('http://tieba.baidu.com' + Article_url)
  46.             jieshao = site.xpath('.//*[@class="threadlist_abs threadlist_abs_onlyline "]')[0].text.strip()
  47.             author = site.xpath('.//*[@class="frs-author-name j_user_card "]')[0].text.strip()
  48.             lastName = site.xpath('.//*[@class="frs-author-name j_user_card "]')[1].text.strip()
  49.             print title, jieshao, Article_url, author, lastName
  51.             item = {}
  53.             item['title'] = title
  54.             item['author'] = author
  55.             item['lastName'] = lastName
  56.             item['reply_date'] = reply_date
  57.             print item
  59.             line = json.dumps(item, ensure_ascii=False)
  60.             print line
  61.             print type(line)
  63.             output.write(line + "\n")
  64.         output.close()
  65.     print 'end'
  67. if __name__ == '__main__':
  68.     main()