国家食品药品监督管理总局

http://app1.sfda.gov.cn/datasearch/face3/dir.html

items.py：添加以下代码

  1. from scrapy import Field
  2. import scrapy
  4. class Sfda1Item(scrapy.Item):
  5.     # define the fields for your item here like:
  6.     data = scrapy.Field()

在spiders目录下新建一个自定义spider

  1. # -*- coding: utf-8 -*-
  2. import scrapy
  3. from scrapy.http import FormRequest
  4. from tutorial.items import Sfda1Item
  5. import urllib
  6. import re
  9. class sfdaSpider(scrapy.Spider):
  10.     name = 'sfda'
  11.     allowed_domains = ['sfda.gov.cn']
  13.     def start_requests(self):
  14.         url = 'http://app1.sfda.gov.cn/datasearch/face3/search.jsp'
  15.         data = {
  16.             'tableId': '32',
  17.             'State': '1',
  18.             'bcId': '124356639813072873644420336632',
  19.             'State': '1',
  20.             'tableName': 'TABLE32',
  21.             'State': '1',
  22.             'viewtitleName': 'COLUMN302',
  23.             'State': '1',
  24.             'viewsubTitleName': 'COLUMN299,COLUMN303',
  25.             'State': '1',
  26.             'curstart': '1',
  27.             'State': '1',
  28.             'tableView': urllib.quote("国产药品商品名"),
  29.             'State': '1',
  30.         }
  31.         yield FormRequest(url=url, formdata=data, meta={'data': data}, callback=self.parseContent)
  33.     def parseContent(self, response):
  35.         for site in response.xpath('//a').re(r'callbackC,\'(.*?)\',null'):
  36.             id = re.search('.+Id=(.*?)$', site).group(1)
  37.             #print id
  39.             url = 'http://app1.sfda.gov.cn/datasearch/face3/content.jsp?tableId=32&tableName=TABLE32&tableView=%B9%FA%B2%FA%D2%A9%C6%B7%C9%CC%C6%B7%C3%FB&Id=' + id
  40.             yield scrapy.Request(url,
  41.                                  headers={
  42.                                      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
  43.                                  },
  44.                                  callback=self.ParseDetail)
  46.         data = response.meta['data']
  47.         data['curstart'] = str(int(data['curstart']) + 1)
  48.         yield FormRequest(url=response.request.url, formdata=data, meta={'data': data}, callback=self.parseContent)
  50.     def ParseDetail(self, response):
  51.         item = dict()
  52.         for site in response.xpath('//table[1]/.//tr')[1:-1]:
  53.             try:
  54.                 if not site.xpath('./td/text()').extract()[0]:
  55.                     continue
  56.                 name = site.xpath('./td/text()').extract()[0]
  57.                 value = re.sub('<.*?>', '', site.xpath('./td')[1].extract()).strip()
  58.                 print name, value
  60.                 item[name] = value
  61.             except Exception, e:
  62.                 print 'error', e
  63.         sfa = Sfda1Item()
  64.         sfa['data'] = item
  65.         yield sfa

在pipelines.py：添加如下代码

  1. import json
  2. import codecs
  4. class JsonWriterPipeline(object):
  6.     def __init__(self):
  7.         self.file = codecs.open('items.json', 'w', encoding='utf-8')
  9.     def process_item(self, item, spider):
  10.         line = json.dumps(dict(item), ensure_ascii=False) + "\n"
  11.         self.file.write(line)
  12.         return item
  14.     def spider_closed(self, spider):
  15.         self.file.close()

settings.py:添加如下代码（启用组件）

  1. ITEM_PIPELINES = {
  2.     'tutorial.pipelines.JsonWriterPipeline': 300,
  3. }

在项目根目录下新建main.py文件,用于调试

  1. from scrapy import cmdline
  2. cmdline.execute('scrapy crawl sfda -L INFO'.split())