from bs4 import BeautifulSoup
import requests

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.8",
    "Connection": "close",
    "Cookie": "_gauges_unique_hour=1; _gauges_unique_day=1; _gauges_unique_month=1; _gauges_unique_year=1; _gauges_unique=1",
    "Referer": "http://www.infoq.com",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER"
}

#url = 'http://www.infoq.com/cn/news'
url = 'http://news.baidu.com/'

def craw(url):
    response = requests.get(url, headers=headers)

    soup = BeautifulSoup(response.text, 'lxml')

    for hotnews in soup.find_all('div', class_='hotnews'):
        for news in hotnews.find_all('a'):
            print(news.text,end=' ')
            print(news.get('href'))


craw(url)

换了个网页爬爬

pip3安装bs4报错 之前安装requests都可以的 报错信息如下
07Exception:
Traceback (most recent call last):
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\site-package
s\pip\_vendor\urllib3\response.py", line 331, in _error_catcher
    yield
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\site-package
s\pip\_vendor\urllib3\response.py", line 413, in read
    data = self._fp.read(amt)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\site-package
s\pip\_vendor\cachecontrol\filewrapper.py", line 62, in read
    data = self.__fp.read(amt)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\http\client.
py", line 449, in read
    n = self.readinto(b)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\http\client.
py", line 493, in readinto
    n = self.fp.readinto(b)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\socket.py",
line 586, in readinto
    return self._sock.recv_into(b)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\ssl.py", lin
e 1009, in recv_into
    return self.read(nbytes, buffer)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\ssl.py", lin
e 871, in read
    return self._sslobj.read(len, buffer)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\ssl.py", lin
e 631, in read
    v = self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\site-package
s\pip\_internal\cli\base_command.py", line 143, in main
    status = self.run(options, args)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\site-package
s\pip\_internal\commands\install.py", line 318, in run
    resolver.resolve(requirement_set)
  File "c:\users\acadsoc\appdata\local\programs\python\python36\lib\site-package