import concurrent.futures import urllib.request import ssl # 定义关于 url 的字典，元素为：（网站名称：url） url_records = {'极客时间':'https://time.geekbang.org/', '百度':'https://www.baidu.com', '京东':'https://www.jd.com', '淘宝':'https://www.taobao.com', '天猫':'https://www.tmall.com' } # 定义方法，访问网址并读取首页内容 def load_url(url, timeout): context = ssl._create_unverified_context() # url是https形式 with urllib.request.urlopen(url, timeout=timeout, context=context) as conn: return conn.read() # 通过线程池，并行执行，并将文件内容写入”网站名称.txt“的文件中 with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: future_to_url = {executor.submit(load_url, url_record[1], 60): url_record for url_record in list(url_records.items())} for future in concurrent.futures.as_completed(future_to_url): url_name = future_to_url[future][0] url_path = future_to_url[future][1] try: data = future.result().decode('utf-8') except Exception as exc: print('%r generated an exception: %s' % (url_path, exc)) else: print('%r page is %d bytes' % (url_path, len(data))) with open(f'{url_name}.txt', 'w') as f: f.write(data)

#请使用并发任务模型同时访问 5 个网站，并将网页的数据存储到不同的文件中 import requests import concurrent.futures as cfs urls = ['http://www.baidu.com', 'https://www.taobao.com', 'https://www.jd.com', 'https://www.tmall.com', 'https://cn.bing.com'] def get_pages(url, timeout): response = requests.get(url=url, timeout=timeout) if response.status_code == 200: return response.text else: return None with cfs.ThreadPoolExecutor(max_workers=5) as exe: futures_dict = {exe.submit(get_pages, url, 10): url for url in urls} for future in cfs.as_completed(futures_dict): url = futures_dict[future] try: data = future.result() except Exception: print(f'{url}数据存储出现异常:{Exception}') else: file_path = './' + url.split('/')[-1] + '.html' with open(file_path, 'w') as f: f.write(data) print(f'{url}存储数据{len(data)}字节')

借鉴学习了下其他同学的，自己写了下， from concurrent.futures import ThreadPoolExecutor, as_completed import urllib.request URLS = [ 'https://time.geekbang.org', 'https://www.taobao.com/', 'https://www.jd.com', 'https://leetcode.cn', 'https://www.zhihu.com' ] def load_url(url, timeout): with urllib.request.urlopen(url, timeout=timeout) as conn: return conn.read() with ThreadPoolExecutor(max_workers=5) as executor: future_to_url = {executor.submit(load_url, url, 60): url for url in URLS} for i,future in enumerate(as_completed(future_to_url)): url = future_to_url[future] try: data = future.result().decode('utf-8') except Exception as exc: print('%r generated an exception: %s' % (url, exc)) else: with open(f'./84/{i+1}.txt', 'a', encoding='utf-8') as f: f.write(data) print('%r page is saved' % (url))

import urllib import concurrent from lxml import etree import os URLS = ['https://time.geekbang.org', 'https://www.csdn.net/', 'https://www.jd.com', 'https://leetcode.cn', 'https://www.zhihu.com'] def load_url(url, timeout): with urllib.request.urlopen(url, timeout=timeout) as conn: return conn.read() with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: future_to_url = {executor.submit(load_url, url, 60): url for url in URLS} for i,future in enumerate(concurrent.futures.as_completed(future_to_url)): url = future_to_url[future] try: data = future.result().decode('utf-8') except Exception as exc: print(f'{url} generated an exception:{exc}') else: rst = etree.HTML(data).xpath('//head/title/text()')[0] print(f'The title of {url} page is "{rst}".') with open(f'{i+1}.txt', 'w') as f: f.write(rst) # os.remove(f'{i+1}.txt')