今天在用Scrapy写代码的时候，对网页的结构也有了大致的分析，再加上之前罗罗攀的思路，初步我是通过专题入口
热门专题
image.png
image.png
专题管理员 （一般粉丝、文章、字数、收获喜欢、这几项数据都非常漂亮）
image.png
image.png
以上红框里的数据项就是我需要爬取的字段
但是以上的思路存在一点的问题：
存在一些简书用户并不是一些热门专题的管理员，但是其人气粉丝量也很高，这个思路可能无法将这些用户爬取下来
进阶的思路
热门专题
专题关注的人
专题关注的人的动态
推荐作者 粉丝信息
image.png
image.png
image.png
优点：
数据大而全，基本包含了99%的用户（个人猜测，不严谨）
缺点：
因为许多用户不止关注一个专题，而且其中包含了大量的新注册用户（数据很多为空），并且也有大量重复数据需要去重
代码部分:
jianshu.py 还在调试阶段，待更新...
# -*- coding: utf-8 -*-
import sys
import json
import requests
import scrapy
import re
from lxml import etree
from scrapy.http import Request
reload(sys)
sys.path.append('..')
sys.setdefaultencoding('utf-8')
class jianshu(scrapy.Spider):
name = 'jianshu'
#topic_category = ['city']
topic_category = ['recommend', 'hot', 'city']
base_url = 'lections?page=%s&order_by=%s'
cookies={
'UM_distinctid': '15b89d53a930-02ab95f11ccae2-51462d15-1aeaa0-15b89d53a9489b',
'CNZZDATA1258679142': '1544557204-1492664886-%7C1493280769',
'_session_id': 'Q3RteU9BeTA3UVh1bHp1d24ydmZJaGdkRDZJblE3SWg3dTlNR2J1WmJ5NS9HNlpOZVg4ZUk0TnNObE5wYXc3SjhYcU5WR0NKZ3RhcE9veFVDU2RNWkpqNE44MWxuVmtoR1ZDVXBFQ29Kc1kzZmd4SVNZakJyWVN4c1RFQXZNTFhmUUtxemVDVWlVU1l3VW92NFpTeEE2Q0ppUVN0QVFEMUpLZjFHdHViR21zZko2b1lFTW9DR08yNDh5Z0pvd0VJRzc4aFBqRnZYbGt6QXlmSzMxdU1QTVFwUVcxdUViaElqZzh2Y1RwcENtSWxWbW5PMUVGZ2UrZ2xVcm1NTlpMK2x2UTdOWlZjUVNPK1dCTERpMnd6U3ZxbXlROENML2VseTRHUTBqbFE1ZUlqN1FqazJJK0tsV1htdEt1bnl5MkhCbHNJTmh1ejFLTW9pYVcrVmx0bit1blNXV1VCQ3JNbHAvK1Z5T1ZvUk5IMVMzR1dUNHBlWFZBamcwYjQxSzBjZVRvMGRZSDRmV0xtTGZHekF1M3V6dGcwMHhpQ24zdmVKelV5eDRFSWZ4QT0tLW1uSXNLakp6SW54SUo0QU16a2dFSkE9PQ%3D%3D--0849c37208f8c573960d857029c7d6a15145c419',
'remember_user_token':'W1szNDgxMjU3XSwiJDJhJDEwJDlSS3VLcFFWMlZzNFJuOFFNS1JQR3UiLCIxNDk0MjEzNDQ3LjYwODEwNzgiXQ%3D%3D--9241542a4e44d55acaf8736a1d57dd0e96ad4e7a',
'_ga': 'GA1.2.2016948485.1492666105',
'_gid': 'GA1.2.382495.1494550475',
'Hm_lpvt_0c0e9d9b1e7d617b3e6842e85b9fb068': '1494550475',
'Hm_lvt_0c0e9d9b1e7d617b3e6842e85b9fb068': '1494213432,1494213612,1494321303,1494387194'
}
headers = {
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept - Language': 'zh - CN, zhq = 0.8',
'Connection': 'close',
'Cookie': 'UM_distinctid=15b89d53a930-02ab95f11ccae2-51462d15-1aeaa0-15b89d53a9489bCNZZDATA1258679142=1544557204-1492664886-%7C1493280769remember_user_token=W1szNDgxMjU3XSwiJDJhJDEwJDlSS3VLcFFWMlZzNFJuOFFNS1JQR3UiLCIxNDk0MjEzNDQ3LjYwODEwNzgiXQ%3D%3D--9241542a4e44d55acaf8736a1d57dd0e96ad4e7a_ga=GA1.2.2016948485.1492666105_gid=GA1.2.824702661.1494486429_gat=1Hm_lvt_0c0e9d9b1e7d617b3e6842e85b9fb068=1494213432,1494213612,1494321303,1494387194Hm_lpvt_0c0e9d9b1e7d617b3e6842e85b9fb068=1494486429_session_id=czl6dzVOeXdYaEplRVdndGxWWHQzdVBGTll6TVg5ZXFDTTI5cmN2RUsvS2Y2d3l6YlkrazZkZWdVcmZDSjFuM2tpMHpFVHRTcnRUVnAyeXhRSnU5UEdhaGMrNGgyMTRkeEJYOE9ydmZ4N1prN1NyekFibkQ5K0VrT3paUWE1bnlOdzJrRHRrM0Z2N3d3d3hCcFRhTWdWU0lLVGpWWjNRdjArZkx1V2J0bGJHRjZ1RVBvV25TYnBQZmhiYzNzOXE3VWNBc25YSS93WUdsTEJFSHVIck4wbVI5aWJrUXFaMkJYdW41WktJUDl6OVNqZ2k0NWpGL2dhSWx0S2FpNzhHcFZvNGdQY012QlducWgxNVhoUEN0dUpCeUI4bEd3OXhiMEE2WEplRmtaYlR6VTdlZXFsaFFZMU56M2xXcWwwbmlZeWhVb0dXKzhxdEtJaFZKaUxoZVpUZEZPSnBGWmF3anFJaFZpTU9Icm4wcllqUFhWSzFpYWF4bTZmSEZ1QXdwRWs3SHNEYmNZelA4VG5zK0wvR0MwZDdodlhZakZ6OWRVbUFmaE5JMTIwOD0tLXVyVEVSeVdOLy9Cak9nVG0zV0hueVE9PQ%3D%3D--ea401e8c501e7b749d593e1627dbaa88ab4befc2',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36',
'Host':'www.jianshu.com',
"X-Requested-With": 'XMLHttpRequest'
}
def get_total_page(self):
#获取专题总页数 包含3个字典的列表 [{"hot": xx}, {"recommend":xx}, {"city": xx}]
total_page_list = []
for order in self.topic_category:
order = order.decode('utf-8')
total_page = 100
dict = {}
for page in range(1, total_page):
url = self.base_url % (page, order)
html = requests.get(url, headers=self.headers).content
selector = etree.HTML(html)
#print html
try:
elements = selector.xpath('//*[@id="list-container"]/div[1]/div/h4/a/text()')[0]
if elements is not Exception:
continue
except Exception :
dict['total_page'] = page - 1
dict['category'] = order
break
total_page_list.append(dict)
return total_page_list
def get_topic_info(self):
#获取专题信息
topic_info_list = []
total_page_list = self.get_total_page()
base_url = self.base_url
for dict in total_page_list:
category = dict['category']
total_page = int(dict['total_page'])
for page in range(1, total_page + 1):
url = base_url % (page, category)
html = requests.get(url, headers=self.headers,cookies=self.cookies).content
selector = etree.HTML(html)
topic_href = selector.xpath('//*[@id="list-container"]')[0]
for href in topic_href:
dict = {}
topic_name = href.xpath('./div/h4/a/text()')[0]
topic_url = "www.jianshu.com" + href.xpath('./div/h4/a/@href')[0]
topic_img_url = href.xpath('./div/a/img/@src')[0]
img_num = topic_img_url.split("/")[5]
dict['topic_name'] = topic_name
dict['topic_url'] = topic_url
#
dict['img_num'] = img_num
topic_info_list.append(dict)
return topic_info_list
def get_topic_admin_info(self):
#获取管理员信息
topic_admin_info_list = []
topic_info_list = self.get_topic_info()
for d in topic_info_list:
img_num = str(d['img_num'])
base_url = "s/editors_and_subscribers" % img_num
base_url_response = requests.get(base_url, headers=self.headers, cookies=self.cookies)
json_data_base = json.loads(base_url_response.text.decode('utf-8'))
editors_total_pages = json_data_base['editors_total_pages']
for page in range(1, int(editors_total_pages) + 1):
if page == 1:
editors = json_data_base['editors']
for editor in editors:
dict = {}
dict['nickname'] = editor['nickname']
dict['slug'] = editor['slug']
topic_admin_info_list.append(dict)
else:
try:
url = "}/editors?page={}".format(img_num, page)
response = requests.get(url,headers=self.headers,cookies=self.cookies)
json_data = json.loads(response.text.decode('utf-8'))
editors = json_data['editors']
for editor in editors:
dict = {}
dict['nickname'] = editor['nickname']
dict['slug'] = editor['slug']
topic_admin_info_list.append(dict)
except Exception:
pass
return topic_admin_info_list
def get_followers_following_list(self):
# 获取管理员粉丝列表
followers_list = []
topic_admin_list = self.get_topic_admin_info()
followers_base_url = "s/%s/followers"
for dict in topic_admin_list:
url = followers_base_url % dict['slug']
headers = self.headers
headers['Referer'] = url
headers['DNT'] = '1'
response = requests.get(url, headers=headers, cookies=self.cookies).content
total_followers = re.fi
Python是一种跨平台的计算机程序设计语言。是一个高层次的结合了解释性、编译性、互动性和面向对象的脚本语言。最初被设计用于编写自动化脚本（shell），随着版本的不断更新和语言新功能的添加，越多被用于独立的、大型项目的开发。
Python的创始人为荷兰人吉多·范罗苏姆（GuidovanRossum）。1989年圣诞节期间，在阿姆斯特丹，Guido为了打发圣诞节的无趣，决心开发一个新的脚本解释程序，作为ABC语言的一种继承。
之所以选中Python（大蟒蛇的意思）作为该编程语言的名字，是取自英国20世纪70年代首播的电视喜剧《蒙提．派森的飞行马戏团》（MontyPython＇sFlyingCircus）。
扩展资料：
python中文就是蟒蛇的意思。在计算机中，它是一种编程语言。Python（英语发音：／ˈpaɪθən／），是一种面向对象、解释型计算机程序设计语言，由GuidovanRossum于1989年底发明，第一个公开发行版发行于1991年。Python语法简洁而清晰，具有丰富和强大的类库。
它常被昵称为胶水语言，它能够把用其他语言制作的各种模块（尤其是C／C＋＋）很轻松地联结在一起。常见的一种应用情形是，使用Python快速生成程序的原型（有时甚至是程序的最终界面），然后对其中有特别要求的部分，用更合适的语言改写。
比如3D游戏中的图形渲染模块，性能要求特别高，就可以用C++重写。1发展历程编辑自从20世纪90年代初Python语言诞生至今，它逐渐被广泛应用于处理系统管理任务和Web编程。Python已经成为最受欢迎的程序设计语言之一。
参考资料：
百度百科-Python
