阳光热线问政平台

http://wz.sun0769.com/index.php/question/questionType?type=4

items.py：添加以下代码 

  1. from scrapy.item import Item, Field
  3. class SunItem(Item):
  4.     number = Field()
  5.     url = Field()
  6.     title = Field()
  7.     content = Field()

在spiders目录下新建一个自定义SunSpider.py

  1. from scrapy.spiders import CrawlSpider, Rule
  2. from scrapy.linkextractors import LinkExtractor
  3. from tutorial.items import SunItem
  4. import scrapy
  5. import urllib
  6. import time
  7. import re
  10. class SunSpider(CrawlSpider):
  11.     name = 'sun0769'
  12.     num = 0
  13.     allow_domain = ['http://wz.sun0769.com/']
  14.     start_urls = ['http://wz.sun0769.com/index.php/question/questionType?type=4'
  15.                   ]
  17.     rules = {
  18.         Rule(LinkExtractor(allow='page'), process_links='process_request', follow=True),
  19.         Rule(LinkExtractor(allow='/html/question/\d+/\d+\.shtml$'), callback='parse_content')
  20.     }
  22.     def process_request(self, links):
  23.         ret=[]
  25.         for link in links:
  26.             try:
  27.                 page = re.search('page=\d*', link.url).group()
  28.                 type = re.search('type=\d+', link.url).group()
  29.                 link.url ='http://wz.sun0769.com/index.php/question/questionType?' + page + "&" + type
  30.             except Exception, e:
  31.                 pass
  32.             ret.append(link)
  33.         return ret
  35.     def parse_content(self, response):
  36.         item = SunItem()
  38.         url = response.url
  40.         title = response.xpath('//*[@class="greyframe"]/div/div/strong/text()')[0].extract().strip()
  42.         number = response.xpath('//*[@class="greyframe"]/div/div/strong/text()')[0].extract().strip().split(':')[-1]
  44.         content = response.xpath('//div[@class="c1 text14_2"]/text()').extract()[0].strip()
  46.         item['url'] = url
  47.         item['title'] = title
  48.         item['number'] = number
  49.         item['content'] = content
  50.         yield item

在pipelines.py：添加如下代码

  1. import json
  2. import codecs
  4. class JsonWriterPipeline(object):
  6.     def __init__(self):
  7.         self.file = codecs.open('items.json', 'w', encoding='utf-8')
  9.     def process_item(self, item, spider):
  10.         line = json.dumps(dict(item), ensure_ascii=False) + "\n"
  11.         self.file.write(line)
  12.         return item
  14.     def spider_closed(self, spider):
  15.         self.file.close()

settings.py:添加如下代码（启用组件）

  1. ITEM_PIPELINES = {
  2.     'tutorial.pipelines.JsonWriterPipeline': 300,
  3. }

window 下调试

在项目根目录下新建main.py文件,用于调试

  1. from scrapy import cmdline
  2. cmdline.execute('scrapy crawl sun0769'.split())