import scrapy class QuotesPagerSpider(scrapy.Spider): name = "quotes_pager" allowed_domains = ["quotes.toscrape.com"] start_urls = ["https://quotes.toscrape.com/page/1/"] def parse(self, response): for quote in response.css("div.quote"): yield { "text": quote.css("span.text::text").get(default="").strip(), "author": quote.css("small.author::text").get(default="").strip(), "page_url": response.url, } next_page = response.css("li.next a::attr(href)").get() if next_page: yield response.follow(next_page, callback=self.parse)