import scrapy class ScrollJsSpider(scrapy.Spider): name = "scroll_js" allowed_domains = ["app.internal.example"] start_urls = ["http://app.internal.example:8000/scroll/"] def start_requests(self): for url in self.start_urls: yield scrapy.Request( url=url, meta={"selenium": True, "selenium_wait_css": "#items li"}, ) def parse(self, response): for entry in response.css("#items li"): yield {"title": entry.css("::text").get()}