pip install beautifulsoup4

from bs4 import BeautifulSoup

html_doc = "
Hello, BeautifulSoup4!"
soup = BeautifulSoup(html_doc, 'html.parser')

# 选择所有的段落标签
paragraphs = soup.find_all('p')

# 选择class为'example'的div标签
example_div = soup.find('div', class_='example')

import requests
from bs4 import BeautifulSoup

# 发送HTTP请求获取页面内容
url = 'https://example.com'
response = requests.get(url)
html_content = response.text

# 使用BeautifulSoup解析HTML
soup = BeautifulSoup(html_content, 'html.parser')

# 提取标题和链接
title = soup.title.text
links = soup.find_all('a')

# 打印结果
print(f"页面标题：{title}")

print("页面链接：")
for link in links:
    print(f"{link.get('href')}")

# 数据处理与异常处理示例
for link in links:
    try:
        # 提取链接文本并去除首尾空白字符
        link_text = link.text.strip()
        
        # 提取链接地址
        link_url = link.get('href')

        # 打印处理后的结果
        print(f"链接文本：{link_text}, 链接地址：{link_url}")

        # 进一步处理数据，比如提取数字
        if link_text.isdigit():
            number = int(link_text)
            print(f"提取到数字：{number}")
    
    except Exception as e:
        # 异常处理，打印异常信息
        print(f"处理链接时发生异常：{e}")

# 使用CSS选择器提取数据
main_content = soup.select_one('#main-content').text

# 使用正则表达式匹配特定模式的数据
import re
pattern = re.compile(r'bd{3}-d{2}-d{4}b')
matches = soup.find_all(text=pattern)

# 定制化解析器示例
from bs4 import SoupStrainer

only_a_tags = SoupStrainer("a")
custom_soup = BeautifulSoup(html_content, 'html.parser', parse_only=only_a_tags)

import requests

# 创建Session对象
session = requests.Session()

# 使用Session发送请求
response = session.get('https://example.com')

impor服务器托管网t fake_useragent
from bs4 import BeautifulSoup
import requests

# 随机生成User-Agent
headers = {'User-Agent': fake_useragent.UserAgent().random}

# 使用IP代理
proxies = {'http': 'http://your_proxy', 'https': 'https://your_proxy'}

response = requests.get('https://example.com', headers=headers, proxies=proxies)

from selenium import webdriver

# 使用Selenium模拟浏览器
driver = webdriver.Chrome()
driver.get('https://example.com')

import requests
from bs4 import BeautifulSoup
from datetime import datetime

# 发送HTTP请求获取新闻页面内容
url = 'https://example-news-website.com'
response = requests.get(url)
html_content = response.text

# 使用BeautifulSoup解析HTML
soup = BeautifulSoup(html_content, 'html.parser')

# 提取新闻信息
news_list = []

for news_item in soup.find_all('div', class_='news-item'):
    try:
        # 提取新闻标题
        title = news_item.find('h2').text.strip()

        # 提取新闻链接
        link = news_item.find('a')['href']

        # 提取发布时间
        time_string = news_item.find('span', class_='publish-time').text.strip()
        publish_time = datetime.strptime(time_string, '%Y-%m-%d %H:%M:%S')

        # 将提取的信息存入字典
        news_info = {'title': title, 'link': link, 'publish_time': publish_time}
        news_list.append(news_info)

    except Exception as e:
        # 异常处理，打印异常信息
        print(f"处理新闻时发生异常：{e}")

# 打印提取的新闻信息
for news_info in news_list:
    print(f"标题：{news_info['title']}")
    print(f"链接：{news_info['link']}")
    print(f"发布时间：{news_info['publish_time']}")
    print("n")