美团App热门商圈团购采集(2)

把上节内容生成的城市信息 items.json改成city_items.json作为第二部分爬虫的启动数据

添加items.py

  1. class MeituanItem(Item):
  2.     data = Field()

创建模板：

  1. scrapy genspider -t basic Meituan_meishi meituan.com

添加以下代码到Meituan_meishi.py

  1. # -*- coding: utf-8 -*-
  2. import scrapy
  3. import codecs
  4. import json
  5. from tutorial.items import MeituanItem
  6. import re
  8. class MeituanMeishiSpider(scrapy.Spider):
  9.     '''
  10.     美食团购页面信息采集
  11.     '''
  12.     name = "Meituan_meishi"
  13.     allowed_domains = ["meituan.com"]
  14.     '''
  15.     start_urls = (
  16.         'http://www.meituan.com/',
  17.     )
  18.     '''
  19.     offset = 0
  20.     def start_requests(self):
  21.         file = codecs.open('city_items.json', 'r', encoding='utf-8')
  22.         for line in file:
  23.             item = json.loads(line)
  24.             cityid = item['data']['cityid']
  25.             latitude = item['data']['latitude']
  26.             longitude= item['data']['longitude']
  28.             lat = round(float(latitude), 6)
  29.             lng= round(float(longitude), 6)
  31.             url = 'http://api.mobile.meituan.com/group/v4/deal/select/city/42/cate/1?sort=defaults&mypos='+ str(lat) +'%2C'+ str(lng) +'&offset=0&limit=15'
  32.             yield scrapy.Request(url,callback=self.parse)
  33.             break
  34.         file.close()
  37.     def parse(self, response):
  38.         '''
  39.         数据存储以及翻页操作
  40.         '''
  41.         item = MeituanItem()
  43.         data = json.loads(response.body)
  44.         item['data']=dict()
  45.         item['data'] = data
  46.         yield item
  48.         offset = re.search('offset=(\d+)',response.request.url).group(1)
  49.         url = re.sub('offset=\d+','offset='+str(int(offset)+15),response.request.url)
  50.         yield scrapy.Request(url,callback=self.parse)

运行：

  1. scrapy runspider tutorial/spiders/Meituan_meishi.py