feat: introduce custom tags extractor

newspaper/article.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 # Much of the code here was forked from https://github.com/codelucas/newspaper
 # Copyright (c) Lucas Ou-Yang (codelucas)
-"""Module providing the Article class for newspaper. The Article class
+"""
+Module providing the Article class for newspaper. The Article class
 abstracts the concept of a news article, providing methods and properties
 to download, parse and analyze said article.
 """
@@ -476,11 +477,16 @@ def parse(self) -> "Article":
         self.meta_description = metadata["description"]
         self.canonical_link = metadata["canonical_link"]
         self.meta_keywords = metadata["keywords"]
-        self.tags = metadata["tags"]
+        self.tags = set(metadata.get("tags") or [])
         self.meta_data = metadata["data"]
 
         self.publish_date = self.extractor.get_publishing_date(self.url, self.doc)
 
+        # Custom tags from new TagsExtractor
+        extracted_tags = self.extractor.get_tags(self.doc)
+        if extracted_tags:
+            self.tags = self.tags.union(extracted_tags)
+
         # Top node in the original documentDOM
         self.top_node = self.extractor.calculate_best_node(self.doc)
         # Off-tree Node containing the top node and any relevant siblings

newspaper/extractors/content_extractor.py

@@ -2,6 +2,7 @@
 from datetime import datetime
 from typing import Dict, Any, List, Optional
 import lxml
+from .tags_extractor import TagsExtractor
 from newspaper import urls
 import newspaper.parsers as parsers
 from newspaper.configuration import Configuration
@@ -33,7 +34,7 @@ class ContentExtractor:
         title_extractor (TitleExtractor): The title extractor object.
         author_extractor (AuthorsExtractor): The authors extractor object.
         pubdate_extractor (PubdateExtractor): The publishing date extractor object.
-        article_body_extractor (ArticleBodyExtractor): The article body
+        atricle_body_extractor (ArticleBodyExtractor): The article body
             extractor object.
         metadata_extractor (MetadataExtractor): The metadata extractor object.
         categories_extractor (CategoryExtractor): The category extractor object.
@@ -46,11 +47,12 @@ def __init__(self, config: Configuration):
         self.title_extractor = TitleExtractor(config)
         self.author_extractor = AuthorsExtractor(config)
         self.pubdate_extractor = PubdateExtractor(config)
-        self.article_body_extractor = ArticleBodyExtractor(config)
+        self.atricle_body_extractor = ArticleBodyExtractor(config)
         self.metadata_extractor = MetadataExtractor(config)
         self.categories_extractor = CategoryExtractor(config)
         self.image_extractor = ImageExtractor(config)
         self.video_extractor = VideoExtractor(config)
+        self.tags_extractor = TagsExtractor(config)
 
     def get_authors(self, doc: lxml.html.Element) -> List[str]:
         """Fetch the authors of the article, return as a list
@@ -138,7 +140,7 @@ def top_node(self) -> lxml.html.Element:
         Returns:
             lxml.html.Element: The top node containing the article text
         """
-        return self.article_body_extractor.top_node
+        return self.atricle_body_extractor.top_node
 
     @property
     def top_node_complemented(self) -> lxml.html.Element:
@@ -147,7 +149,7 @@ def top_node_complemented(self) -> lxml.html.Element:
         Returns:
             lxml.html.Element: deepcopy version of the top node, cleaned
         """
-        return self.article_body_extractor.top_node_complemented
+        return self.atricle_body_extractor.top_node_complemented
 
     def calculate_best_node(
         self, doc: lxml.html.Element
@@ -164,9 +166,9 @@ def calculate_best_node(
             lxml.html.Element: the article top element
             (most probable container of the article text), or None
         """
-        self.article_body_extractor.parse(doc)
+        self.atricle_body_extractor.parse(doc)
 
-        return self.article_body_extractor.top_node
+        return self.atricle_body_extractor.top_node
 
     def get_videos(
         self, doc: lxml.html.Element, top_node: lxml.html.Element
@@ -181,3 +183,13 @@ def get_videos(
             List[str]: list of video urls
         """
         return self.video_extractor.parse(doc, top_node)
+
+    def get_tags(self, doc: lxml.html.Element) -> List[str]:
+        """
+        A wrapper that calls our custom TagsExtractor to parse tags from the DOM.
+
+        Returns:
+            List[str]: A list of tag strings.
+        """
+        return self.tags_extractor.parse(doc)
+

newspaper/extractors/tags_extractor.py

@@ -0,0 +1,60 @@
+import lxml.html
+from typing import List
+from newspaper.configuration import Configuration
+import newspaper.parsers as parsers
+
+class TagsExtractor:
+    """
+    A custom extractor that attempts to find "tags" for the article.
+
+    Looks for containers like:
+      1) <div id="articleTag"> with links <a class="lnk"> ...,
+      2) <div class="meta-tags"> with links <a rel="tag"> ...,
+      3) <div class="tags-links"> (common pattern in some themes).
+
+    Then extracts the text of each link and collects them as a list of tags.
+    """
+
+    def __init__(self, config: Configuration):
+        self.config = config
+        self.tags: List[str] = []
+
+    def parse(self, doc: lxml.html.HtmlElement) -> List[str]:
+        """
+        Searches for tags in any container that matches:
+          - <div id="articleTag">,
+          - <div class="meta-tags">,
+          - <div class="tags-links">.
+
+        Inside those containers, it looks for <a> elements that have either
+        class="lnk" or rel="tag". For each <a>, we strip its text
+        and if non-empty, add to the result list.
+
+        Returns:
+            List[str]: A list of tag strings (words/labels).
+        """
+        # Combine possible container paths with an XPath "union"
+        container_nodes = doc.xpath(
+            '//div[@id="articleTag"]'
+            ' | //div[contains(@class,"meta-tags")]'
+            ' | //div[contains(@class,"tags-links")]'
+        )
+        if not container_nodes:
+            self.tags = []
+            return self.tags
+
+        result = []
+        for container in container_nodes:
+            # We'll look for links that match (class="lnk") or (rel="tag") ...
+            link_nodes = container.xpath('.//a[contains(@class,"lnk") or @rel="tag"]')
+
+            # Alternatively, if you wanted *all* <a> tags in these containers:
+            # link_nodes = container.xpath('.//a')
+
+            for link_node in link_nodes:
+                text = parsers.get_text(link_node).strip()
+                if text:
+                    result.append(text)
+
+        self.tags = result
+        return self.tags