腾讯招聘

http://hr.tencent.com/position.php

items.py：添加以下代码 

  1. from scrapy.item import Item, Field
  3. class TencentItem(Item):
  4.     title = Field()
  5.     catalog = Field()
  6.     workLocation = Field()
  7.     recruitNumber = Field()
  8.     duty = Field()
  9.     Job_requirement= Field()
  10.     url = Field()
  11.     publishTime = Field()

在spiders目录下新建一个自定义tencent_info.py

  1. # -*- coding:utf-8 -*-
  2. from scrapy.selector import Selector
  3. from scrapy.linkextractors import LinkExtractor
  4. from scrapy.spiders import CrawlSpider, Rule
  5. import re
  6. from tutorial.items import TencentItem
  9. class TencentSpider(CrawlSpider):
  10.     name = "tengxun_info"
  11.     allowed_domains = ["tencent.com"]
  12.     start_urls = [
  13.         "http://hr.tencent.com/position.php"
  14.     ]
  16.     rules = [
  17.         Rule(LinkExtractor(allow=("start=\d+"))),
  18.         Rule(LinkExtractor(allow=("position_detail\.php")), follow=True, callback='parse_item')
  19.     ]
  21.     def parse_item(self,response):
  23.         item =TencentItem()
  25.         title = response.xpath('//*[@id="sharetitle"]/text()')[0].extract()
  26.         workLocation = response.xpath('//*[@class="lightblue l2"]/../text()')[0].extract()
  27.         catalog = response.xpath('//*[@class="lightblue"]/../text()')[0].extract()
  28.         recruitNumber = response.xpath('//*[@class="lightblue"]/../text()').re('(\d+)')[0]
  29.         duty_pre = response.xpath('//*[@class="squareli"]')[0].extract()
  30.         duty = re.sub('<.*?>','',duty_pre)
  32.         Job_requirement_pre = response.xpath('//*[@class="squareli"]')[1].extract()
  33.         Job_requirement = re.sub('<.*?>','',Job_requirement_pre)
  35.         item['title']=title
  36.         item['url']=response.url
  37.         item['workLocation']=workLocation
  38.         item['catalog']=catalog
  39.         item['recruitNumber']=recruitNumber
  40.         item['duty']=duty
  41.         item['Job_requirement']=Job_requirement
  43.         yield item

在pipelines.py：添加如下代码

  1. import json
  2. import codecs
  4. class JsonWriterPipeline(object):
  6.     def __init__(self):
  7.         self.file = codecs.open('items.json', 'w', encoding='utf-8')
  9.     def process_item(self, item, spider):
  10.         line = json.dumps(dict(item), ensure_ascii=False) + "\n"
  11.         self.file.write(line)
  12.         return item
  14.     def spider_closed(self, spider):
  15.         self.file.close()

settings.py:添加如下代码（启用组件）

  1. ITEM_PIPELINES = {
  2.     'tutorial.pipelines.JsonWriterPipeline': 300,
  3. }

在项目根目录下新建main.py文件,用于调试

  1. from scrapy import cmdline
  2. cmdline.execute('scrapy crawl tengxun_info'.split())