爬蟲確實已經登入進去了，因為我爬取個人資訊是可以抓出來的，但是下圖的這個網址抓不出來：
網址是：http://write.blog.csdn.net/postlist，就是你的csdn後台。
我貼下程式碼吧，py2.7的
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
class CSDN(object):
def __init__(self, headers):
self.session = requests.Session()
self.headers = headers
def get_webflow(self):
url = 'http://passport.csdn.net/account/login'
response = self.session.get(url=url, headers=self.headers)
soup = BeautifulSoup(response.text, 'html.parser')
lt = soup.find('input', {'name': 'lt'})['value']
execution = soup.find('input', {'name': 'execution'})['value']
soup.clear()
return (lt, execution)
def login(self, account, password):
self.username = account
self.password = password
lt, execution = self.get_webflow()
data = {
'username': account,
'password': password,
'lt': lt,
'execution': execution,
'_eventId': 'submit'
}
url = 'http://passport.csdn.net/account/login'
response = self.session.post(url=url, headers=self.headers, data=data)
if (response.status_code == 200):
print('正常')
else:
print('异常')
def func(self):
headers1={
'Host':'write.blog.csdn.net',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
}
response=self.session.get(url='http://write.blog.csdn.net/postlist',headers=headers1,allow_redirects=False)
print response.text
if __name__ == '__main__':
headers = {
'Host': 'passport.csdn.net',
'Origin': 'http://passport.csdn.net',
'Referer':'http://passport.csdn.net/account/login',
'Upgrade-Insecure-Requests':'1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36',
}
csdn = CSDN(headers=headers)
account = ''
password = ''
csdn.login(account=account, password=password)
csdn.func()
上面的程式碼輸出是
正常
<html><head><title>Object moved</title></head><body>
<h2>Object moved to <a href="https://passport.csdn.net?service=http://write.blog.csdn.net/postlist">here</a>.</h2>
</body></html>
因為這個地址返回的是一個302跳轉，你要根據返回header的Location繼續請求，再分析返回的內容繼續處理，瀏覽器幫你做了這些302跳轉和執行返回的js等內容，手工抓取就需要自己處理．
直接用cookie即可