Pagination-With-Python/infinite_scroll_html.py at main · oxylabs/Pagination-With-Python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# Handling pages with load more with HTML response
import requests
from bs4 import BeautifulSoup
import math


def process_pages():
    index_page = 'https://techinstr.myshopify.com/collections/all'
    url = 'https://techinstr.myshopify.com/collections/all?page={}'

    session = requests.session()
    response = session.get(index_page)
    soup = BeautifulSoup(response.text, "lxml")
    count_element = soup.select_one('.filters-toolbar__product-count')
    count_str = count_element.text.replace('products', '')
    count = int(count_str)
    # Process page 1 data here
    page_count = math.ceil(count/8)
    for page_numer in range(2, page_count+1):
        response = session.get(url.format(page_numer))
        soup = BeautifulSoup(response.text, "lxml")
        first_product = soup.select_one('.product-card:nth-child(1) > a > span')
        print(first_product.text.strip())


if __name__ == '__main__':
    process_pages()