除了使用phantomjs,selenium之外,怎么爬取多页评论,这两个都太慢了。例如http://music.163.com/#/song?i... 的 评论。
webapi都是http://music.163.com/weapi/v1...,
每页20个评论,怎么获取下一页的评论,param是加密的,post都不知道post什么数据
在网上有一种方式只能获取
def aes_encrypt(self, text, secKey):
pad = 16 - len(text) % 16
text = text + pad * chr(pad)
encryptor = AES.new(secKey, 2, '0102030405060708')
ciphertext = encryptor.encrypt(text)
ciphertext = base64.b64encode(ciphertext)
return ciphertext
def rsa_encrypt(self, text, pubKey, modulus):
text = text[::-1]
rs = int(text.encode('hex'), 16) ** int(pubKey, 16) % int(modulus, 16)
return format(rs, 'x').zfill(256)
def create_secret_key(self, size):
return (''.join(map(lambda xx: (hex(ord(xx))[2:]), os.urandom(size))))[0:16]
def encrypt(self, url):
song_id = url[url.find('=') + 1:]
print song_id
url = 'http://music.163.com/weapi/v1/resource/comments/R_SO_4_' + str(song_id) + '/?csrf_token='
headers = {'Cookie': 'appver=1.5.0.75771;', 'Referer': 'http://music.163.com/song?id=27901109'}
text = {'username': '', 'password': '', 'rememberLogin': 'true', 'page': "2"}
modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7'
nonce = '0CoJUm6Qyw8W8jud'
pub_key = '010001'
text = json.dumps(text)
sec_key = self.create_secret_key(16)
enc_Text = self.aes_encrypt(self.aes_encrypt(text, nonce), sec_key)
enc_sec_key = self.rsa_encrypt(sec_key, pub_key, modulus)
data = {'params': enc_Text, 'encSecKey': enc_sec_key}
req = requests.post(url, headers=headers, data=data)
data = req.json()
这种只能获取前10条评论,后面的评论怎么获取,急求大神指导啊