from itemadapter import ItemAdapter from scrapy.spidermiddlewares.base import BaseSpiderMiddleware class CatalogSpiderMiddleware(BaseSpiderMiddleware): def get_processed_item(self, item, response): adapter = ItemAdapter(item) adapter["source_url"] = response.url if response else None adapter["processed_by_spider_middleware"] = True return item