페이지 넘기기 처리 방법

https://example.com/products?page=1
https://example.com/products?page=2
https://example.com/catalog/page/3

import requests
from bs4 import BeautifulSoup

all_products = []

for page in range(1, 100):
    html = requests.get(f"https://example.com/products?page={page}").text
    soup = BeautifulSoup(html, "html.parser")
    cards = soup.select(".product-card")

    if not cards:
        break

    for card in cards:
        all_products.append(card.select_one(".title").get_text(strip=True))

from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

url = "https://example.com/products"
seen_urls = set()

while url and url not in seen_urls:
    seen_urls.add(url)
    soup = BeautifulSoup(requests.get(url).text, "html.parser")

    for card in soup.select(".product-card"):
        print(card.select_one(".title").get_text(strip=True))

    next_link = soup.select_one("a[rel='next'], a.next")
    url = urljoin(url, next_link["href"]) if next_link and next_link.get("href") else None

import requests

offset = 0
limit = 24
products = []

while True:
    data = requests.get(
        "https://example.com/api/products",
        params={"offset": offset, "limit": limit},
    ).json()

    batch = data.get("items", [])
    if not batch:
        break

    products.extend(batch)
    offset += len(batch)

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto("https://example.com/products")

    previous_count = 0
    for _ in range(40):
        page.mouse.wheel(0, 4000)
        page.wait_for_timeout(1500)

        current_count = page.locator(".product-card").count()
        if current_count == previous_count:
            break
        previous_count = current_count

    print(page.locator(".product-card").count())
    browser.close()

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto("https://example.com/products")

    while page.locator("button.load-more").is_visible():
        before = page.locator(".product-card").count()
        page.locator("button.load-more").click()
        page.wait_for_function(
            "(count) => document.querySelectorAll('.product-card').length > count",
            before,
        )

    browser.close()

/api/products?offset=40&limit=20

{
  "items": [],
  "pageInfo": {
    "hasNextPage": true,
    "endCursor": "eyJpZCI6MTAwfQ=="
  }
}

import requests

cursor = None

while True:
    params = {"limit": 50}
    if cursor:
        params["after"] = cursor

    data = requests.get("https://example.com/api/products", params=params).json()
    for item in data.get("items", []):
        print(item["name"])

    page_info = data.get("pageInfo", {})
    if not page_info.get("hasNextPage"):
        break

    cursor = page_info.get("endCursor")