Python の HTTP クライアント モジュール urllib および urllib3 の使用方法

import urllib.parse

url = "http://www.baidu.com"
parsed = urllib.parse.urlparse(url)
print(parsed)
# 输出：ParseResult(scheme='http', netloc='www.baidu.com', path='', params='', query='', fragment='') #返回的是一个parseresult类型的元组对象，可以通过属性或者索引来获取值

from urllib.parse import urlunparse
data=['http','www.baidu.com','index.html','user','a=6','comment']
print(urlunparse(data)) #构造一个完整的URL

#output
#http://www.baidu.com/index.html;user?a=6#comment

from urllib.parse import urlsplit

result = urlsplit('http://www.baidu.com/index.html;user?id=5#comment')
print(result)
print(result.query)
# output
# SplitResult(scheme='http', netloc='www.baidu.com', path='/index.html;user', query='id=5', fragment='comment')
# id=5

from urllib.parse import urlsplit,urlunsplit
data=['http','www.baidu.com','index.html','a=5','comment']
result=urlunsplit(data)
print(result)

#output
#http://www.baidu.com/index.html?a=5#comment

import urllib.parse
dic = {'name':'melon','age':18}
data = urllib.parse.urlencode(dic)

print(data)     #age=18&name=melon

from urllib.parse import urlencode,parse_qs,urlsplit
params = {'username':'zs','password':'123'}
base_url='http://www.baidu.com'
url=base_url+'?'+urlencode(params) #将字典转化为get参数

query=urlsplit(url).query  #获去URL的query参数条件
print(parse_qs(query))  #根据获取的GET参数转换为字典格式

#output
#{'username': ['zs'], 'password': ['123']}

from urllib.parse import quote
key='中文'
url='https://www.baidu.com/s?key='+quote(key)
print(url)

#output
#https://www.baidu.com/s?key=%E4%B8%AD%E6%96%87

from urllib.parse import quote, urlsplit, unquote

key = '中文'
url = 'https://www.baidu.com/s?key=' + quote(key)
print(url)
unq = urlsplit(url).query.split('=')[1]  # 获取参数值

print(unquote(unq))  # 解码参数

import urllib.parse

url = "http://www.baidu.com"
new_path = urllib.parse.urljoin(url, "index.html")
print(new_path)
# 输出：<a href="http://www.baidu.com/index.html" rel="external nofollow"   target="_blank">http://www.baidu.com/index.html</a>

file_name = urllib.request.urlretrieve('http://www.baidu.com','%s/baidu.html'%BASE_DIR)

urllib.request.urlopen(url, data=None, [timeout, ]*, cafile=None, capath=None, cadefault=False, context=None)

import urllib.request

req = urllib.request.urlopen('http://www.baidu.com')
print(req.read().decode())
print(req.getheaders())  # 以列表元祖对的形式返回响应头信息
print(req.getheader('Content-Type'))  # 返回响应头中的Content-Type值
print(req.info())  # 返回网页的头信息
print(req.info()['Content-Type'])  # 返回响应头中的Content-Type值

for line in urlopen('https://。。.html'):
    line = line.decode('utf-8')  # Decoding the binary data to text.
    if 'EST' in line or 'EDT' in line:  # look for Eastern Time
        print(line)

import urllib.request
import urllib.parse

dic = {'name':'melon','age':18}
data = urllib.parse.urlencode(dic)

req = urllib.request.urlopen('http://127.0.0.1:8000/index?%s'%data) #通过urlopen方法访问拼接好的url

content = req.read()

import urllib.request
import urllib.parse
import json

dic = {'name':'melon','age':18}
data = urllib.parse.urlencode(dic)

req = urllib.request.Request('http://127.0.0.1:8000/index', data.encode()) #encode：将url编码类型的请求数据转变为bytes类型

opener = urllib.request.urlopen(req)
content = json.loads(opener.read().decode()) #read()方法是读取返回bytes数据内容，decode转换后为str

# Request对象实例化
req   = urllib.request.Request(url, data=None, headers={},origin_req_host=None,unverifiable=False, method=None)

from urllib import request

url='http://httpbin.org/post'
headers={
     'User-Agent':'Mozilla/5.0 (compatible; MSIE 5.5; Windows NT)',
     'Host':'httpbin.org'
 }  #定义头信息

dict={'name':'germey'}

data = bytes(parse.urlencode(dict),encoding='utf-8')
req = request.Request(url=url,data=data,headers=headers,method='POST')
#req.add_header('User-Agent','Mozilla/5.0 (compatible; MSIE 8.4; Windows NT') #也可以request的方法来添加
resp = request.urlopen(req) 
print(resp.read().decode())

build.opener(handler1,handler2…)

# 将这个opener设置为全局的opener，之后所有的，不管是opener.open()还是urlopen() 发送请求，都将使用自定义
request.install_opener(opener)
resp = request.urlopen(url)

from urllib.request import HTTPPasswordMgrWithDefaultRealm,HTTPBasicAuthHandler,build_opener
from urllib.error import URLError

username='username'
passowrd='password'
url='http://localhost'
p=HTTPPasswordMgrWithDefaultRealm() #构造密码管理实例
p.add_password(None,url,username,passowrd) #添加用户名和密码到实例中
auth_handler=HTTPBasicAuthHandler(p) #传递密码管理实例构建一个验证实例
opener=build_opener(auth_handler)  #构建一个Opener
try:
    result=opener.open(url)  #打开链接，完成验证，返回的结果是验证后的页面内容
    html=result.read().decode('utf-8')
    print(html)
except URLError as e:
    print(e.reason)

from urllib import request
  
url = 'http://httpbin.org/ip'
proxy = {'http': '218.18.232.26:80', 'https': '218.18.232.26:80'}
proxies = request.ProxyHandler(proxy)  # 创建代理处理器
opener = request.build_opener(proxies)  # 创建opener对象
  
resp = opener.open(url)
print(resp.read().decode())

from http import cookiejar
from urllib import request
  
cookie = cookiejar.CookieJar() # 创建一个cookiejar对象
cookies = request.HTTPCookieProcessor(cookie) # 使用HTTPCookieProcessor创建cookie处理器
opener = request.build_opener(cookies) # 并以它为参数创建Opener对象
resp = opener.open('https://www.baidu.com') # 使用这个opener来发起请求
  
# 查看之前的cookie对象，则可以看到访问百度获得的cookie
for i in cookie:
    print(i)

import http.cookiejar,urllib.request
fielname='cookies.txt'
cookie=http.cookiejar.MozillaCookieJar(filename=fielname) #创建保存cookie的实例，保存浏览器类型的Mozilla的cookie格式
handler=urllib.request.HTTPCookieProcessor(cookie) #构建一个handler
opener=urllib.request.build_opener(handler) #构建Opener
response=opener.open('http://www.baidu.com') #请求
cookie.save(ignore_discard=True,ignore_expires=True)

import http.cookiejar,urllib.request
#fielname='cookiesLWP.txt'
#cookie=http.cookiejar.LWPCookieJar(filename=fielname) #LWP格式的cookies
cookie=http.cookiejar.LWPCookieJar()
cookie.load('cookiesLWP.txt',ignore_discard=True,ignore_expires=True)

handler=urllib.request.HTTPCookieProcessor(cookie) #构建一个handler
opener=urllib.request.build_opener(handler) #构建Opener
response=opener.open('http://www.baidu.com') #请求
print(response.read().decode('utf-8'))

import ssl
from urllib import request
 
context = ssl._create_unverified_context()
res = urllib.request.urlopen(request, context=context)

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

from urllib import request,error

try:
    response=request.urlopen('https://hehe,com/index')
except error.URLError as e:
    print(e.reason)  #如果网页不存在不会抛出异常，而是返回捕获的异常错误的原因(Not Found)

import socket
import urllib.request
import urllib.error
try:
    response=urllib.request.urlopen('https://www.baidu.com',timeout=0.001)
except urllib.error.URLError as e:
    print(e.reason)
    if isinstance(e.reason,socket.timeout):
        print('time out')

from urllib import request,error

try:
    response=request.urlopen('http://cuiqingcai.com/index.htm')
except error.HTTPError as e:  #先捕获子类异常
    print(e.reason,e.code,e.headers,sep='\n')
except error.URLError as e:  #再捕获父类异常

    print(e.reason)
else:
    print('request successfully')

User-agent: *
Disallow: /
Allow: /public/

#禁止所有爬虫
User-agent: *
Disallow: /

#允许所有爬虫访问任何目录,另外把文件留空也可以
User-agent: *
Disallow:

#禁止所有爬虫访问某那些目录
User-agent: *
Disallow: /home/
Disallow: /tmp/

#只允许某一个爬虫访问
User-agent: BaiduSpider
Disallow:
User-agent: *
Disallow: /

urllib.robotparser.RobotFileParser(url='')

from urllib.robotparser import RobotFileParser

rp = RobotFileParser()  #创建对象
rp.set_url('https://www.cnblogs.com/robots.txt') #设置robots.txt连接，也可以在创建对象时指定
rp.read()  #读取和解析文件
print(rp.can_fetch('*','https://i.cnblogs.com/EditPosts.aspx?postid=9170312&update=1')) #判断链接是否可以被抓取

pip install urllib3

class urllib3.poolmanager.PoolManager（num_pools = 10，headers = None，** connection_pool_kw ）

import urllib3

data = json.dumps({'abc': '123'})
http = urllib3.PoolManager(num_pools=5, headers={'User-Agent': 'ABCDE'})
  
resp1 = http.request('GET', 'http://www.baidu.com', body=data)
resp2 = http.urlopen('GET', 'http://www.baidu.com', body=data)
  
print(resp2.data.decode())

import urllib3
import json
  
data = json.dumps({'abc': '123'})
http = urllib3.PoolManager(num_pools=5, headers={'User-Agent': 'ABCDE'})
  
resp1 = http.request('POST', 'http://www.httpbin.org/post', body=data,timeout=5,retries=5)
#resp2 = http.urlopen('POST', 'http://www.httpbin.org/post', body=data,timeout=5,retries=5)
  
print(resp1.data.decode())

def __init__(self, proxy_url, num_pools=10, headers=None,proxy_headers=None, **connection_pool_kw):

import urllib3
import json
 
data = {'abc': '123'}
proxy = urllib3.ProxyManager('http://50.233.137.33:80', headers={'connection': 'keep-alive'})
resp1 = proxy.request('POST', 'http://www.httpbin.org/post', fields=data)
print(resp1.data.decode())