import requests from bs4 import BeautifulSoup url = 'https://www.tianqi.com/shenyang/7/' def getHTMLtext(url): """请求获得网页内容""" headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"} try: r = requests.get(url, timeout = 30, headers = headers) r.raise_for_status() r.encoding = r.apparent_encoding print("成功访问") return r.text except: print("访问错误") return" " def get_content(html): """处理得到有用信息保存数据文件""" final = "" # 初始化一个列表保存数据 bs = BeautifulSoup(html, "html.parser") # 创建BeautifulSoup对象 body = bs.body ul = body.find('ul', {'class': 'weaul'}) # 找到ul标签且class = weaul li = ul.find_all('li') # 找到左右的li标签 i = 0 # 控制爬取的天数 for day in li: # 遍历找到的每一个li if i < 30 and i > 0: temp = "" # 临时存放每天的数据 date = day.find(attrs={'class':'fl'}).string # 得到日期 temp += date + " " inf = day.find_all(attrs={'class':'weaul_z'}) # 找出li下面的weaul_z,提取第一个weaul_z标签的值，即天气 temp += inf[0].string + " " tem = inf[1].find_all('span') tem_low = tem[0].string # 找到最低温度 tem_high = tem[1].string # 找到最高温度 temp += tem_low+'~'+tem_high+'℃' + "\r" final += temp i = i + 1 return final if __name__ == '__main__': html_text = getHTMLtext(url) data = get_content(html_text) with open('weather.txt', 'w', encoding='UTF-8') as f: f.write(str(data))

from multiprocessing import Process,Queue import random class Producer(Process): def __init__(self,queue): super().__init__() self.queue = queue def run(self):# 启动任务 self.create() def create(self): num = random.randint(1,10000) self.queue.put(num) class Consumer(Process): def __init__(self,queue): super().__init__() self.queue = queue def run(self):# 启动任务 self.get_result() def get_result(self): random_num = self.queue.get() if random_num % 7 ==0: print("The number is",random_num,", is divisible by 7.") else: print("Generated number", random_num, "is not divisible by 7.") if __name__ == '__main__': queue = Queue() process1 = Producer(queue) process2 = Consumer(queue) process1.start() process2.start() # 启动进程 process1.join() # 结束进程 process2.join()

import requests from bs4 import BeautifulSoup import pandas as pd import matplotlib.pyplot as plt # 抓取深圳的天气情况 url = "http://www.weather.com.cn/weather/101280601.shtml" response = requests.get(url) response.encoding = 'UTF-8' # print(response) soup = BeautifulSoup(response.text, 'html.parser') # print(soup) # 解析网页，获取日期、最高温度和最低温度 dates = [item.get_text() for item in soup.select('.t li h1')] dates = [item.get_text() for item in soup.select('.t li h1')] high_temps = [int(item.get_text().replace('℃', '')) for item in soup.select('.t li .tem span')] low_temps = [int(item.get_text().replace('℃', '')) for item in soup.select('.t li .tem i')] # 将数据保存到 DataFrame 中 df = pd.DataFrame({ 'date': dates, 'high_temp': high_temps, 'low_temp': low_temps }) # 将 DataFrame 保存到 CSV 文件中 df.to_csv('shenzhen_weather.csv', index=False) # 从 CSV 文件中读取 DataFrame df = pd.read_csv('shenzhen_weather.csv') # 绘制最高温度和最低温度的折线图 plt.rcParams['font.sans-serif'] = ['Arial Unicode MS'] plt.plot(df['date'], df['high_temp'], label='High Temperature') plt.plot(df['date'], df['low_temp'], label='Low Temperature') plt.xlabel('Date') plt.ylabel('Temperature (℃)') plt.title('Shenzhen Weather') plt.legend() plt.show()

# 没找到降水的信息在哪 import requests from bs4 import BeautifulSoup import os import json def getHtml(url): header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36' } try: r = requests.get(url, headers=header) r.raise_for_status() return r.text except: print('请求失败') def getContent(): html = getHtml('https://www.tianqi.com/shanghai/7/') soup = BeautifulSoup(html, 'html.parser') # 找到展示天气的标签 body = soup.body ul = body.find('ul', {'class': 'weaul'}) li = ul.find_all('li') # 每天的天气信息放在li标签中 wea_list = [] for i in li: date = i.find('div', {'class': 'weaul_q'}).text.split('\n')[0] wea_wrapper = i.find('div', {'class': 'weaul_z'}) wea = wea_wrapper.text temp = wea_wrapper.find_next_sibling('div', {'class': 'weaul_z'}).text wea_list.append({ 'date': date, 'temp': temp, 'wea': wea }) return wea_list path = os.getcwd() BASE_PATH = f"{path}/tong_zheng/2023_12_04_homework/" # 绝对路径基本 def saveContent(): with open(f"{BASE_PATH}weather.json", 'w', encoding='utf-8') as f: content = getContent() json.dump(content, f, ensure_ascii=False) saveContent()

有两个地方没明白， 第一，获取数据中的文字，为什么从浏览器F12中的"预览"项中获取，而不是"响应"项 第二，代码中 data = {"ids":[100008801],"with_first_articles":False} 中的 ids 是什么意思啊，为什么不写成 id 或者 abc 呢

发现可以通过添加头信息来绕过检查：除了添加User-Agent外，还需要添加 Origin，加上就可以请求的通了

老师，目前按视频中的接口，已经走不通了返回451了

import requests import datetime from matplotlib import pyplot def get_data(accesskey, citycode, date): url = f"http://service.envicloud.cn:8082/v2/weatherhistory/{accesskey}/{citycode}/{date}" payload = "" headers = {'cache-control': "no-cache"} response = requests.request("GET", url, data=payload, headers=headers) return response.json() def save_data(data): date = data['date'] tem_max = data['tem_max'] tem_min = data['tem_min'] filename = '/Users/mike/Desktop/zero-basics-python/0/weather_data.txt' with open(filename, 'a') as file: file.write(f'{date},{tem_max},{tem_min}\n') def show_data(): filename = '/Users/mike/Desktop/zero-basics-python/0/weather_data.txt' dates, highs, lows = [], [], [] with open(filename, 'r') as file: line = file.readline() while line: line_list = line.split(',') dates.append(line_list[0]) highs.append(line_list[1]) lows.append(line_list[2]) line = file.readline() fig = pyplot.figure(dpi=128, figsize=(10, 6)) pyplot.plot(dates, highs, c='red', alpha=0.5) pyplot.plot(dates, lows, c='blue', alpha=0.5) pyplot.fill_between(dates, highs, lows, facecolor='blue', alpha=0.1) pyplot.title('Daily high and low tempertures', fontsize=24) pyplot.xlabel('', fontsize=16) fig.autofmt_xdate() pyplot.ylabel('Temperature', fontsize=16) pyplot.tick_params(axis='both', which='major', labelsize=16) pyplot.show() today = datetime.date.today() for i in range(1,8): format_today = today.strftime('%Y%m%d') data = get_data(accesskey='xxx', citycode='xxx', date=format_today) save_data(data) today = today - datetime.timedelta(days=i) show_data()

# 请求所在城市7天的天气数据, 并存入本地文本 # https://www.tianqi.com/shenyang/7/ import requests from bs4 import BeautifulSoup urlstr = "https://www.tianqi.com/shanghai/7/" header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36" } r = requests.get(urlstr, headers=header) bs = BeautifulSoup(r.text, "html.parser") # bs.body html的主体 # print(bs.body) body = bs.body ul = body.find("ul", {"class": "weaul"}) # print(ul) li = ul.find_all("li") # print(li) temp = "" for day in li: print(day) date = day.find("span", {"class": "fr"}).string temp += date temp += " " temp += day.find("div", {"class": "weaul_z"}).string temp += "\n" print(temp) with open("weather.txt", "w", encoding="UTF-8") as f: f.write(temp)

# 用format()将结果打印输出 def print_data(final_list, num): print("{:^10}\t{:^8}\t{:^8}\t{:^8}\t{:^8}".format('日期', '天气', '最高温度', '最低温度', '风级')) for i in range(num): final = final_list[i] print("{:^10}\t{:^8}\t{:^8}\t{:^8}\t{:^8}".format(final[0], final[1], final[2], final[3], final[4])) # 测试 if __name__ == '__main__': # 中国天气网 url = "http://www.weather.com.cn/weather/101190101.shtml" # 调用函数，爬取网站数据，整合成列表 html = getHTMLText(url) final_list = get_data(html) print_data(final_list, 7) # 将未来7天的日期、天气、最高温度、最低温度、风级数据写入文件 with open("天气.txt", mode='w', encoding='utf-8') as f: f.write(str(final_list))