import scrapy class HomesSpider(scrapy.Spider): name = "homes" allowed_domains = ["property.example"] start_urls = ["https://property.example/real-estate/"] def parse(self, response): for card in response.css("article.listing"): href = card.css("a.detail::attr(href)").get() preview_price = card.css("span.price::text").get(default="").strip() preview_title = card.css("h2::text").get(default="").strip() if href: yield response.follow( href, callback=self.parse_listing, cb_kwargs={ "preview_price": preview_price, "preview_title": preview_title, }, ) next_href = response.css("a.next::attr(href)").get() if next_href: yield response.follow(next_href, callback=self.parse) def parse_listing(self, response, preview_price, preview_title): yield { "listing_id": response.url.rstrip("/").split("/")[-1].replace(".html", ""), "title": response.css("h1::text").get(default=preview_title).strip(), "price": response.css(".price::text").get(default=preview_price).strip(), "city": response.css(".city::text").get(default="").strip(), "beds": response.css(".beds::text").get(default="").strip(), "url": response.url, }