Web Scraping

Requests vs Aiohttp

BeautifulSoup

pip install requests beautifulsoup4 lxml

import requests

from bs4 import BeautifulSoup

url = 'https://en.wikipedia.org/wiki/Beautiful_Soup_(HTML_parser)'
soup = BeautifulSoup(requests.get(url).content, 'lxml')

Selenium

Install dependencies

pip install selenium webdriver_manager

Start the webdriver

from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.chrome.service import Service as ChromiumService
from webdriver_manager.chrome import ChromeDriverManager

options = ChromeOptions()
options.add_argument('--start-maximized')

path = ChromeDriverManager().install()
driver = webdriver.Chrome(service=ChromiumService(path), options=options)