calibre-web: dtrpg-metadata Add DMs Guild and fix cover downloads

DriveThruRPG contains multiple stores, one of which is the DMs Guild
for Dungeons and Dragons. Items that are only for sale on the Guild
don't show up on DTRPG searches, and vice versa.

A lot of the images end up being redirected to an eventual file, but
Calibre-web doesn't follow redirects when saving the cover. Instead, we
follow them up front and return the eventual URL.

root/drivethrurpg.py

@@ -13,7 +13,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, cast
 from urllib.parse import quote
 from lxml import html
 import requests
@@ -24,16 +24,132 @@ from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
 log = logger.create()
 
+HEADERS = {"User-Agent": "Not Evil Browser", "accept-encoding": "gzip"}
+
+
+class DMSGuild(Metadata):
+    __name__ = "DMSGuild"
+    __id__ = "dmsguild"
+    DESCRIPTION = "DM's Guild"
+    META_URL = "https://www.dmsguild.com"
+    BASE_URL = f"{META_URL}/includes/ajax/search_autocomplete_jquery.php?term="
+    QUERY_PARAMS = "&json=true"
+    HEADERS = {"User-Agent": "Not Evil Browser", "accept-encoding": "gzip"}
+
+    def search(self, query: str, generic_cover: str = "", locale: str = "en"):
+        if not self.active:
+            return None
+
+        title_tokens = list(self.get_title_tokens(query, strip_joiners=False))
+        if title_tokens:
+            tokens = [quote(t.encode("utf-8")) for t in title_tokens]
+            query = "%20".join(tokens)
+
+        matches = _do_dtrpg_search(
+            query=f"{self.BASE_URL}{query}{self.QUERY_PARAMS}",
+            source=MetaSourceInfo(
+                id=self.__id__,
+                description=self.DESCRIPTION,
+                link=self.META_URL,
+            ),
+        )
+
+        return matches
+
 
 class DriveThruRpg(Metadata):
     __name__ = "DriveThruRPG"
     __id__ = "drivethrurpg"
     DESCRIPTION = "DriveThru RPG"
-    META_URL = "https://www.drivethrurpg.com/"
+    META_URL = "https://www.drivethrurpg.com"
-    BASE_URL = "https://www.drivethrurpg.com/includes/ajax/search_autocomplete_jquery.php?term="
+    BASE_URL = f"{META_URL}/includes/ajax/search_autocomplete_jquery.php?term="
     QUERY_PARAMS = "&json=true"
-    HEADERS = {"User-Agent": "Not Evil Browser", "accept-encoding": "gzip"}
 
+    def search(
+        self, query: str, generic_cover: str = "", locale: str = "en"
+    ) -> Optional[List[MetaRecord]]:
+        if not self.active:
+            return None
+        
+        title_tokens = list(self.get_title_tokens(query, strip_joiners=False))
+        if title_tokens:
+            tokens = [quote(t.encode("utf-8")) for t in title_tokens]
+            query = "%20".join(tokens)
+
+        matches = _do_dtrpg_search(
+            query=f"{self.BASE_URL}{query}{self.QUERY_PARAMS}",
+            source=MetaSourceInfo(
+                id=self.__id__,
+                description=self.DESCRIPTION,
+                link=self.META_URL,
+            ),
+        )
+
+        return matches
+
+
+def _do_dtrpg_search(query: str, source: MetaSourceInfo) -> List[MetaRecord]:
+    try:
+        log.info(f"Requesting data from: {query}")
+        result = requests.get(
+            query,
+            headers=HEADERS,
+        )
+        result.raise_for_status()
+    except Exception as e:
+        log.warning(e)
+        return list()
+
+    # If there are no hits we see a single element being returned with the easiest
+    # identifier being the link.
+    results_list: list = result.json()
+    if len(results_list) == 1 and results_list[0]["link"] == "#":
+        log.info("No results found")
+        return list()
+
+    # Since we'll go on to do N further requests for more information,
+    # we'll cut it off at the first five results here. Any sufficiently well
+    # populated search by title should be enough
+    results: List[MetaRecord] = list()
+    for r in results_list[0:5]:
+        assert isinstance(r, dict)
+        match = _fetch_dtrpg_search_result(result=r, source=source)
+
+        identifiers = {}
+        identifiers[source.id] = match.id
+
+        match.identifiers = identifiers
+
+        results.append(match)
+
+    return results
+
+
+def _fetch_dtrpg_search_result(result: Dict, source: MetaSourceInfo) -> MetaRecord:
+    match = MetaRecord(
+        id=result["name"],
+        title=result["name"],
+        authors=[],
+        url=result.get("link", ""),
+        source=source,
+    )
+
+    try:
+        details_result = requests.get(
+            result["link"],
+            headers=HEADERS,
+        )
+        details_result.raise_for_status()
+    except Exception as e:
+        log.warning(e)
+        return match
+
+    _parse_dtrpg_result(details_result.content, match)
+
+    return match
+
+
+def _parse_dtrpg_result(content: bytes, match: MetaRecord):
     AUTHORS_XPATH = "//div[@class='widget-information-wrapper']//div[@class='widget-information-item-title' and contains(text(), 'Author(s)')]"
     RULE_SYSTEMS_XPATH = "//div[@class='widget-information-wrapper']//div[@class='widget-information-item-title' and contains(text(), 'Rule System(s)')]"
     PUBLISHER_XPATH = "//div[@class='widget-information-wrapper-2']//div[@class='widget-information-title' and contains(text(), 'Publisher')]"
@@ -41,118 +157,65 @@ class DriveThruRpg(Metadata):
     DESCRIPTION_XPATH = "//div[contains(@class,'prod-content')]//text()"
     IMAGE_PROP_XPATH = "//meta[@itemprop='image']/@content"
 
-    def search(
+    data = html.fromstring(content)
-        self, query: str, generic_cover: str = "", locale: str = "en"
-    ) -> Optional[List[MetaRecord]]:
-        val = list()
-        if self.active:
-            title_tokens = list(self.get_title_tokens(query, strip_joiners=False))
-            if title_tokens:
-                tokens = [quote(t.encode("utf-8")) for t in title_tokens]
-                query = "%20".join(tokens)
 
-            try:
+    # Use the big text field as description as the meta tag is very short
-                result = requests.get(
+    description_field = data.xpath(DESCRIPTION_XPATH)
-                    f"{DriveThruRpg.BASE_URL}{query}{DriveThruRpg.QUERY_PARAMS}",
+    assert isinstance(description_field, List)
-                    headers=DriveThruRpg.HEADERS,
+    if description_field is not None:
-                )
+        match.description = "".join(description_field).strip()  # type: ignore
-                result.raise_for_status()
-            except Exception as e:
-                log.warning(e)
-                return None
 
-            # Since we'll do on to do N further requests for more information,
+    product_url = data.xpath(URL_PROP_XPATH)
-            # we'll cut it off at the first five results here. Any sufficiently well
+    assert isinstance(product_url, List)
-            # populated search by title should be enough
+    if product_url is not None and len(product_url) > 0:
-            for r in result.json()[0:5]:
+        match.url = cast(str, product_url[0])
-                assert isinstance(r, dict)
-                match = self._parse_search_result(
-                    result=r, generic_cover=generic_cover, locale=locale
-                )
-                val.append(match)
-        return val
 
-    def _parse_search_result(
+        # We can get a better ID from the URL
-        self, result: Dict, generic_cover: str, locale: str
+        regex = r".*\/product\/(\d+)\/.*"
-    ) -> MetaRecord:
+        matches = re.findall(regex, match.url)
-        match = MetaRecord(
+        if len(matches) > 0:
-            id=result["name"],
+            match.id = matches[0]
-            title=result["name"],
+
-            authors=[],
+    image_url = data.xpath(IMAGE_PROP_XPATH)
-            url=result.get("link", ""),
+    assert isinstance(image_url, List)
-            source=MetaSourceInfo(
+    if image_url is not None and len(image_url) > 0:
-                id=self.__id__,
+        # Calibre web doesn't follow redirects and reports some covers as an error
-                description=DriveThruRpg.DESCRIPTION,
+        log.info(f"Cover URL is {image_url[0]}")
-                link=DriveThruRpg.META_URL,
+        r = requests.head(image_url[0], allow_redirects=True)
-            ),
+        log.info(f"After following redirects, it is {r.url}") 
+        match.cover = cast(str, r.url)
+
+    # Find authors
+    for div in cast(List, data.xpath(AUTHORS_XPATH)):
+        # Just bring in elements that look like they might be authors.
+        authors = list(
+            filter(
+                lambda x: re.match(r"^\w[\w\s]+$", x),
+                div.getnext().xpath(".//text()"),
+            )
         )
+        match.authors = authors
 
-        try:
+    # Use rule systems as tags
-            details_result = requests.get(
+    match.tags = ["RPG"]
-                result["link"],
+    for div in cast(list, data.xpath(RULE_SYSTEMS_XPATH)):
-                headers=DriveThruRpg.HEADERS,
+        rule_systems = list(
+            filter(
+                lambda x: len(x.strip()) > 0,
+                div.getnext().xpath(".//text()"),
             )
-            details_result.raise_for_status()
+        )
-        except Exception as e:
+        match.tags.extend(rule_systems)
-            log.warning(e)
-            return match
 
-        data = html.fromstring(details_result.content)
+    for div in cast(List, data.xpath(PUBLISHER_XPATH)):
-
+        publisher_link = div.getnext().xpath(".//a")
-        # Use the big text field as description as the meta tag is very short
+        # Sometimes we get a link, other times it's text in a different element.
-        description_field = data.xpath(self.DESCRIPTION_XPATH)
+        if publisher_link is not None and len(publisher_link) > 0:
-        if description_field is not None:
+            match.publisher = publisher_link[0].text_content().strip()
-            match.description = "".join(description_field).strip()
+        else:
-
+            publisher_name = div.getnext().xpath(
-        product_url = data.xpath(self.URL_PROP_XPATH)
+                ".//div[@class='widget-information-item-title']"
-        if product_url is not None and len(product_url) > 0:
-            match.url = product_url[0]
-
-            # We can get a better ID from the URL
-            regex = r".*\/product\/(\d+)\/.*"
-            matches = re.findall(regex, match.url)
-            if len(matches) > 0:
-                match.id = matches[0]
-
-        image_url = data.xpath(self.IMAGE_PROP_XPATH)
-        if image_url is not None and len(image_url) > 0:
-            match.cover = image_url[0]
-
-        # Find authors
-        for div in data.xpath(self.AUTHORS_XPATH):
-            # Just bring in elements that look like they might be authors.
-            authors = list(
-                filter(
-                    lambda x: re.match(r"^\w[\w\s]+$", x),
-                    div.getnext().xpath(".//text()"),
-                )
             )
-            match.authors = authors
+            match.publisher = publisher_name[0].text_content().strip()
 
-        # Use rule systems as tags
+    return match
-        match.tags = ["RPG"]
-        for div in data.xpath(self.RULE_SYSTEMS_XPATH):
-            rule_systems = list(
-                filter(
-                    # lambda x: re.match(r"^\w[()\w\s]+$", x),
-                    lambda x: len(x.strip()) > 0,
-                    div.getnext().xpath(".//text()"),
-                )
-            )
-            match.tags.extend(rule_systems)
-
-        for div in data.xpath(self.PUBLISHER_XPATH):
-            publisher_link = div.getnext().xpath(".//a")
-            # Sometimes we get a link, other times it's text in a different element.
-            if publisher_link is not None and len(publisher_link) > 0:
-                match.publisher = publisher_link[0].text_content().strip()
-            else:
-                publisher_name = div.getnext().xpath(
-                    ".//div[@class='widget-information-item-title']"
-                )
-                match.publisher = publisher_name[0].text_content().strip()
-
-        # match.publishedDate = result.get("store_date", result.get("date_added"))
-        match.identifiers = {"drivethrurpg": match.id}
-
-        return match