import scrapy from scrapy.exceptions import CloseSpider from scrapy.http import JsonRequest class CountriesSpider(scrapy.Spider): name = "countries" allowed_domains = ["countries.trevorblades.com"] url = "https://countries.trevorblades.com/" custom_settings = { "FEED_EXPORT_ENCODING": "utf-8", } async def start(self): query = """ query ContinentCountries($code: ID!) { continent(code: $code) { countries { code name capital } } } """.strip() body = { "operationName": "ContinentCountries", "query": query, "variables": {"code": "EU"}, } yield JsonRequest( url=self.url, data=body, callback=self.parse, ) def parse(self, response): body = response.json() if body.get("errors"): raise CloseSpider("GraphQL errors returned.") data = body.get("data") or {} continent = data.get("continent") or {} countries = continent.get("countries") or [] if not countries: raise CloseSpider("No countries returned.") for row in countries: yield { "code": row.get("code"), "name": row.get("name"), "capital": row.get("capital"), }