Python 3 抓取网页资源的 N 种方法

短命女 2022-08-08 00:49 161阅读 0赞

1、最简单

import urllib.request  
response = urllib.request.urlopen( ' http://python.org/ ')  
html = response.read()

2、使用 Request

import urllib.request  
  
req = urllib.request.Request( ' http://python.org/ ')  
response = urllib.request.urlopen(req)  
the\_page = response.read()

3、发送数据

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
import urllib.parse  
import urllib.request  
  
url =  ' http://localhost/login.php '  
user\_agent =  ' Mozilla/4.0 (compatible; MSIE 5.5; Windows NT) '  
values = \{  
' act ' :  ' login ',  
' login\[email\] ' :  ' yzhang@i9i8.com ',  
' login\[password\] ' :  ' 123456 '  
\}  
  
data = urllib.parse.urlencode(values)  
req = urllib.request.Request(url, data)  
req.add\_header( ' Referer ',  ' http://www.python.org/ ')  
response = urllib.request.urlopen(req)  
the\_page = response.read()  
  
print(the\_page.decode( " utf8 "))

![复制代码][copycode.gif]

4、发送数据和header

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
import urllib.parse  
import urllib.request  
  
url =  ' http://localhost/login.php '  
user\_agent =  ' Mozilla/4.0 (compatible; MSIE 5.5; Windows NT) '  
values = \{  
' act ' :  ' login ',  
' login\[email\] ' :  ' yzhang@i9i8.com ',  
' login\[password\] ' :  ' 123456 '  
\}  
headers = \{  ' User-Agent ' : user\_agent \}  
  
data = urllib.parse.urlencode(values)  
req = urllib.request.Request(url, data, headers)  
response = urllib.request.urlopen(req)  
the\_page = response.read()  
  
print(the\_page.decode( " utf8 "))

![复制代码][copycode.gif]

5、http 错误

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
import urllib.request  
  
req = urllib.request.Request( ' http://www.python.org/fish.html ')  
try:  
urllib.request.urlopen(req)  
except urllib.error.HTTPError as e:  
print(e.code)  
print(e.read().decode( " utf8 "))

![复制代码][copycode.gif]

6、异常处理1

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
from urllib.request  import Request, urlopen  
from urllib.error  import URLError, HTTPError  
req = Request( " http://twitter.com/ ")  
try:  
response = urlopen(req)  
except HTTPError as e:  
print( ' The server couldn\\'t fulfill the request. ')  
print( ' Error code:  ', e.code)  
except URLError as e:  
print( ' We failed to reach a server. ')  
print( ' Reason:  ', e.reason)  
else:  
print( " good! ")  
print(response.read().decode( " utf8 "))

![复制代码][copycode.gif]

7、异常处理2

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
from urllib.request  import Request, urlopen  
from urllib.error  import URLError  
req = Request( " http://twitter.com/ ")  
try:  
response = urlopen(req)  
except URLError as e:  
if hasattr(e,  ' reason '):  
print( ' We failed to reach a server. ')  
print( ' Reason:  ', e.reason)  
elif hasattr(e,  ' code '):  
print( ' The server couldn\\'t fulfill the request. ')  
print( ' Error code:  ', e.code)  
else:  
print( " good! ")  
print(response.read().decode( " utf8 "))

![复制代码][copycode.gif]

8、HTTP 认证

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
import urllib.request  
  
\#  create a password manager   
password\_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()  
  
\#  Add the username and password.   
\#  If we knew the realm, we could use it instead of None.   
top\_level\_url =  " https://cms.tetx.com/ "  
password\_mgr.add\_password(None, top\_level\_url,  ' yzhang ',  ' cccddd ')  
  
handler = urllib.request.HTTPBasicAuthHandler(password\_mgr)  
  
\#  create "opener" (OpenerDirector instance)   
opener = urllib.request.build\_opener(handler)  
  
\#  use the opener to fetch a URL   
a\_url =  " https://cms.tetx.com/ "  
x = opener.open(a\_url)  
print(x.read())  
  
\#  Install the opener.   
\#  Now all calls to urllib.request.urlopen use our opener.   
urllib.request.install\_opener(opener)  
  
a = urllib.request.urlopen(a\_url).read().decode( ' utf8 ')  
print(a)

![复制代码][copycode.gif]

9、使用代理

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
import urllib.request  
  
proxy\_support = urllib.request.ProxyHandler(\{ ' sock5 ':  ' localhost:1080 '\})  
opener = urllib.request.build\_opener(proxy\_support)  
urllib.request.install\_opener(opener)  
  
  
a = urllib.request.urlopen( " http://g.cn ").read().decode( " utf8 ")  
print(a)

![复制代码][copycode.gif]

10、超时

![复制代码][copycode.gif]

\# ! /usr/bin/env python3   
  
import socket  
import urllib.request  
  
\#  timeout in seconds   
timeout = 2  
socket.setdefaulttimeout(timeout)  
  
\#  this call to urllib.request.urlopen now uses the default timeout   
\#  we have set in the socket module   
req = urllib.request.Request( ' http://twitter.com/ ')  
a = urllib.request.urlopen(req).read()  
print(a)

![复制代码][copycode.gif]

@see  [http://docs.python.org/release/3.2/howto/urllib2.html][http_docs.python.org_release_3.2_howto_urllib2.html]

[copycode.gif]: /images/20220805/da1faeb6ff4e44b2b8a9e26596b0033e.png
[http_docs.python.org_release_3.2_howto_urllib2.html]: http://docs.python.org/release/3.2/howto/urllib2.html