from urllib.parse import urlparse

import scrapy


class NewsSpider(scrapy.Spider):
    name = "news"

    async def start(self):
        archive_url = getattr(
            self,
            "archive_url",
            "https://news.example.net/archive/",
        )
        self.max_pages = int(getattr(self, "max_pages", "2"))
        self.pages_seen = 0

        host = urlparse(archive_url).hostname
        if host:
            self.allowed_domains = [host]

        yield scrapy.Request(archive_url, callback=self.parse)

    def parse(self, response):
        self.pages_seen += 1

        yield from response.follow_all(
            response.css("article.archive-entry h2 a::attr(href)"),
            callback=self.parse_article,
        )

        if self.pages_seen < self.max_pages:
            next_page = response.css("nav.pagination a.next::attr(href)").get()
            if next_page:
                yield response.follow(next_page, callback=self.parse)

    def parse_article(self, response):
        yield {
            "headline": response.css("h1::text").get(default="").strip(),
            "published": response.css("time::attr(datetime)").get(default="").strip(),
            "summary": response.css("div.article-summary p::text").get(default="").strip(),
            "url": response.url,
        }