Der Crawler hat sich tatsächlich angemeldet, da ich persönliche Informationen crawlen und erfassen kann, aber die URL im Bild unten kann nicht erfasst werden:
Die URL lautet: http://write.blog.csdn.net/postlist, was Ihr CSDN-Backend ist.
Ich werde den Code posten, er ist für py2.7
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
class CSDN(object):
def __init__(self, headers):
self.session = requests.Session()
self.headers = headers
def get_webflow(self):
url = 'http://passport.csdn.net/account/login'
response = self.session.get(url=url, headers=self.headers)
soup = BeautifulSoup(response.text, 'html.parser')
lt = soup.find('input', {'name': 'lt'})['value']
execution = soup.find('input', {'name': 'execution'})['value']
soup.clear()
return (lt, execution)
def login(self, account, password):
self.username = account
self.password = password
lt, execution = self.get_webflow()
data = {
'username': account,
'password': password,
'lt': lt,
'execution': execution,
'_eventId': 'submit'
}
url = 'http://passport.csdn.net/account/login'
response = self.session.post(url=url, headers=self.headers, data=data)
if (response.status_code == 200):
print('正常')
else:
print('异常')
def func(self):
headers1={
'Host':'write.blog.csdn.net',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
}
response=self.session.get(url='http://write.blog.csdn.net/postlist',headers=headers1,allow_redirects=False)
print response.text
if __name__ == '__main__':
headers = {
'Host': 'passport.csdn.net',
'Origin': 'http://passport.csdn.net',
'Referer':'http://passport.csdn.net/account/login',
'Upgrade-Insecure-Requests':'1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36',
}
csdn = CSDN(headers=headers)
account = ''
password = ''
csdn.login(account=account, password=password)
csdn.func()
Die Ausgabe des obigen Codes ist
正常
<html><head><title>Object moved</title></head><body>
<h2>Object moved to <a href="https://passport.csdn.net?service=http://write.blog.csdn.net/postlist">here</a>.</h2>
</body></html>
因为这个地址返回的是一个302跳转，你要根据返回header的Location继续请求，再分析返回的内容继续处理，浏览器帮你做了这些302跳转和执行返回的js等内容，手工抓取就需要自己处理．
直接用cookie即可