需要加异常的应该就只有一个地方:requests.get()发送网页请求的时候。其它地方不涉及IO。也不涉及数据类型变化,不用做数据类型判断。
由于不能访问wiki,所以网页改了成了国内的。
-- ps: 和0.2s比起来太慢了。
# -*- encoding -*-
'''
py 3.6
sulime
'''
import concurrent.futures
import threading
import requests
import time
now = lambda: time.perf_counter()
def download_one(url):
try:
req = requests.get(url)
req.raise_for_status()
print('Read {} from {}'.format(len(req.content), url))
except:
print('404')
def download_all(sites):
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
executor.map(download_one, sites)
def main():
sites = [
'https://www.baidu.com/',
'https://pypi.org/',
'https://www.sina.com.cn/',
'https://www.163.com/',
'https://news.qq.com/',
'http://www.ifeng.com/',
'http://www.ce.cn/',
'https://news.baidu.com/',
'http://www.people.com.cn/',
'http://www.ce.cn/',
'https://news.163.com/',
'http://news.sohu.com/'
]
start = now()
download_all(sites)
print('Download {} sites in {} s'.format(len(sites), now() - start))
if __name__ == '__main__':
main()
# Read 2443 from https://www.baidu.com/
# Read 6216 from https://news.qq.com/
# Read 699004 from https://www.163.com/
# Read 250164 from http://www.ifeng.com/
# Read 579572 from https://www.sina.com.cn/
# Read 107530 from http://www.ce.cn/
# Read 165901 from http://www.people.com.cn/
# Read 107530 from http://www.ce.cn/
# Read 210816 from https://news.163.com/
# Read 74060 from https://news.baidu.com/
# Read 174553 from http://news.sohu.com/
# Read 19492 from https://pypi.org/
# Download 12 sites in 2.8500169346527673 s
# [Finished in 3.6s]
展开