美团App热门商圈团购采集(1)

环境：

针对美团版本5.4

美团App热门商圈团购采集(1) - 图1

在tutorial项目下

新建一个spider

  1. scrapy genspider -t basic Meituan_City meituan.com

编辑items.py

  1. class MeituanCity(Item):
  2.     data = Field()

编辑 Meituan_City.py

  1. # -*- coding: utf-8 -*-
  2. import scrapy
  3. import json
  4. from tutorial.items import MeituanCity
  6. class MeituanCitySpider(scrapy.Spider):
  7.     name = "Meituan_City"
  8.     allowed_domains = ["meituan.com"]
  9.     start_urls = (
  10.         'http://api.mobile.meituan.com/group/v1/city/list?show=all',
  11.     )
  13.     def parse(self, response):
  14.         data = json.loads(response.body)
  15.         for item in data['data']:
  16.             cityId = item['id']
  17.             # http://api.mobile.meituan.com/group/v2/area/list?cityId=42&spatialFields=center
  18.             url = 'http://api.meituan.com/group/v2/area/list?cityId=%s&spatialFields=center' % cityId
  19.             print url
  20.             yield scrapy.Request(
  21.                 url,
  22.                 callback=self.Parse_Geo,
  23.                 meta={'item': item}
  24.             )
  25.             break
  28.     def Parse_Geo(self, response):
  29.         print response.url
  30.         data = json.loads(response.body)
  31.         metaitem = response.meta['item']
  33.         # 商区信息
  34.         subareasinfo = dict()
  35.         if 'data' in data:
  36.             if 'subareasinfo' in data['data']:
  37.                 for item in data['data']['subareasinfo']:
  38.                     subareasinfo[item['id']] = item
  40.         if 'data' in data:
  41.             if 'areasinfo' in data['data']:
  42.                 for line in data['data']['areasinfo']:
  43.                     # 行政区
  44.                     districtName = line['name']
  45.                     districtId = line['id']
  46.                     for tmp in line['subareas']:
  47.                         # 商圈信息
  48.                         area = subareasinfo[tmp]
  49.                         center = area['center']
  50.                         center = center.replace('POINT(', '').replace(')', '').split()
  51.                         if len(center) > 1:
  52.                             lat = center[1]
  53.                             lng = center[0]
  54.                         longitude = None
  55.                         latitude = None
  56.                         try:
  57.                             longitude = str(int(float(lng) * 1000000))
  58.                             latitude = str(int(float(lat) * 1000000))
  59.                         except:
  60.                             pass
  61.                         Item = MeituanCity()
  62.                         Item['data'] =dict()
  63.                         geoItem=Item['data'] 
  64.                         # 城市信息
  65.                         geoItem['cityid'] = metaitem['id']
  66.                         geoItem['cityname'] = metaitem['name']
  67.                         # 行政区
  68.                         geoItem['districtId'] = districtId
  69.                         geoItem['districtName'] = districtName
  70.                         # 商圈
  71.                         geoItem['SubAreaId'] = area['id']
  72.                         geoItem['secondArea'] = area['name']
  73.                         # 经纬度
  74.                         geoItem['longitude'] = longitude
  75.                         geoItem['latitude'] = latitude
  76.                         yield Item

此时运行：

  1. scrapy runspider tutorial/spiders/Meituan_City.py