JSON看起来像python类型（列表，字典）的字符串。
在之前的文章中，我们说到了怎么用response的方法，获取到网页正确解码后的字符串。如果还有不懂的，可以先阅读 Python爬虫（三）Requests库 。接下来以有道翻译为例子，说说怎么通过网页解码后的字符串，提取到翻译结果。
再结合上述有道翻译的例子，得到字典类型的返回结果，并提取出来翻译结果。
将上述例子的dict_json换成str字符串，再写入文本中。
执行完上述的程序，会得到一个fanyi.txt的文件，其结果如下：{"type": "ZH_CN2EN", "errorCode": 0, "elapsedTime": 1, "translateResult": [[{"src": "\u4eba\u751f\u82e6\u77ed\uff0c\u6211\u7528python", "tgt": "Life is too short, I use python"}]]}。这样子的一份文档，中文部分显示的是二进制，且格式非常不利于阅读，这并不是我们想要的结果。好在json.dumps()为我们提供的两个方法，以帮助我们更好阅读文档。
1.ensure_ascii，能够让中文显示成中文；
2.indent，能够让下一行在第一行的基础上空格。
其用法如下：
生成Request的时候与一般的网页是相同的，提交Request后scrapy就会下载相应的网页生成Response，这时只用解析response.body按照解析json的方法就可以提取数据了。代码示例如下（以京东为例，其中的parse_phone_price和parse_commnets是通过json提取的，省略部分代码）：# -*- coding: utf-8 -*-
from scrapy.spiders import Spider, CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from jdcom.items import JdPhoneCommentItem, JdPhoneItem
from scrapy import Request
from datetime import datetime
import json
import logging
import re
logger = logging.getLogger(__name__)
class JdPhoneSpider(CrawlSpider):
name = "jdPhoneSpider"
start_urls = ["http://list.jd.com/list.html?cat=9987,653,655"]
rules = (
Rule(
LinkExtractor(allow=r"list\.html\?cat\=9987,653,655\&page\=\d+\&trans\=1\&JL\=6_0_0"),
callback="parse_phone_url",
follow=True,
),
)
def parse_phone_url(self, response):
hrefs = response.xpath("//div[@id='plist']/ul/li/div/div[@class='p-name']/a/@href").extract()
phoneIDs = []
for href in hrefs:
phoneID = href[14:-5]
phoneIDs.append(phoneID)
commentsUrl = "http://sclub.jd.com/productpage/p-%s-s-0-t-3-p-0.html" % phoneID
yield Request(commentsUrl, callback=self.parse_commnets)
def parse_phone_price(self, response):
phoneID = response.meta['phoneID']
meta = response.meta
priceStr = response.body.decode("gbk", "ignore")
priceJson = json.loads(priceStr)
price = float(priceJson[0]["p"])
meta['price'] = price
phoneUrl = "http://item.jd.com/%s.html" % phoneID
yield Request(phoneUrl, callback=self.parse_phone_info, meta=meta)
def parse_phone_info(self, response):
pass
def parse_commnets(self, response):
commentsItem = JdPhoneCommentItem()
commentsStr = response.body.decode("gbk", "ignore")
commentsJson = json.loads(commentsStr)
comments = commentsJson['comments']
for comment in comments:
commentsItem['commentId'] = comment['id']
commentsItem['guid'] = comment['guid']
commentsItem['content'] = comment['content']
commentsItem['referenceId'] = comment['referenceId']
# 2016-09-19 13:52:49 %Y-%m-%d %H:%M:%S
datetime.strptime(comment['referenceTime'], "%Y-%m-%d %H:%M:%S")
commentsItem['referenceTime'] = datetime.strptime(comment['referenceTime'], "%Y-%m-%d %H:%M:%S")
commentsItem['referenceName'] = comment['referenceName']
commentsItem['userProvince'] = comment['userProvince']
# commentsItem['userRegisterTime'] = datetime.strptime(comment['userRegisterTime'], "%Y-%m-%d %H:%M:%S")
commentsItem['userRegisterTime'] = comment.get('userRegisterTime')
commentsItem['nickname'] = comment['nickname']
commentsItem['userLevelName'] = comment['userLevelName']
commentsItem['userClientShow'] = comment['userClientShow']
commentsItem['productColor'] = comment['productColor']
# commentsItem['productSize'] = comment['productSize']
commentsItem['productSize'] = comment.get("productSize")
commentsItem['afterDays'] = int(comment['days'])
images = comment.get("images")
images_urls = ""
if images:
for image in images:
images_urls = image["imgUrl"] + ""
commentsItem['imagesUrl'] = images_urls
yield commentsItem
commentCount = commentsJson["productCommentSummary"]["commentCount"]
goodCommentsCount = commentsJson["productCommentSummary"]["goodCount"]
goodCommentsRate = commentsJson["productCommentSummary"]["goodRate"]
generalCommentsCount = commentsJson["productCommentSummary"]["generalCount"]
generalCommentsRate = commentsJson["productCommentSummary"]["generalRate"]
poorCommentsCount = commentsJson["productCommentSummary"]["poorCount"]
poorCommentsRate = commentsJson["productCommentSummary"]["poorRate"]
phoneID = commentsJson["productCommentSummary"]["productId"]
priceUrl = "http://p.3.cn/prices/mgets?skuIds=J_%s" % phoneID
meta = {
"phoneID": phoneID,
"commentCount": commentCount,
"goodCommentsCount": goodCommentsCount,
"goodCommentsRate": goodCommentsRate,
"generalCommentsCount": generalCommentsCount,
"generalCommentsRate": generalCommentsRate,
"poorCommentsCount": poorCommentsCount,
"poorCommentsRate": poorCommentsRate,
}
yield Request(priceUrl, callback=self.parse_phone_price, meta=meta)
pageNum = commentCount / 10 + 1
for i in range(pageNum):
commentsUrl = "http://sclub.jd.com/productpage/p-%s-s-0-t-3-p-%d.html" % (phoneID, i)
yield Request(commentsUrl, callback=self.parse_commnets)
