From 23dbeb8106a06ccfdd4c41b5d6490e9b5e7f19f4 Mon Sep 17 00:00:00 2001
From: Ruowang Sun <sunrw@umich.edu>
Date: Sun, 11 Dec 2022 11:58:50 -0500
Subject: [PATCH] [Callin] Add new extractor

---
 youtube_dl/extractor/callin.py | 43 +++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/callin.py b/youtube_dl/extractor/callin.py
index 8e576cde5..6a0965dca 100644
--- a/youtube_dl/extractor/callin.py
+++ b/youtube_dl/extractor/callin.py
@@ -2,39 +2,60 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
+from ..utils import traverse_obj
 
 class CallinIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?callin\.com/episode/(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://(?:www\.)?callin\.com/episode/(?:[^/#?-]+-)*(?P<id>[^/#?-]+)'
     _TESTS = [{
         'url': 'https://www.callin.com/episode/fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
-        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'md5': '2b17390cd9f80fc3cac530883f6e3d6e',
         'info_dict': {
-            'id': '42',
+            'id': 'PrumRdSQJW',
             'ext': 'mp4',
             'title': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
+            'description': 'Or, why the government doesn’t like SpaceX',
         }
     }, {
         'url': 'https://www.callin.com/episode/episode-81-elites-melt-down-over-student-debt-lzxMidUnjA',
-        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'md5': 'bd6d6b762e0d5c20cd6090e3d27534a8',
         'info_dict': {
-            'id': '42',
+            'id': 'lzxMidUnjA',
             'ext': 'mp4',
             'title': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
+            'description': 'Let’s talk todays episode about the primary election shake up in NYC and the elites melting down over student debt cancelation.',
         }
     }]
 
+    def _search_nextjs_data(self, webpage, video_id, *, transform_source=None, fatal=True, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', fatal=fatal, **kw),
+            video_id, transform_source=transform_source, fatal=fatal)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        # webpage_json = self._download_json(url, video_id)
 
-        # TODO more code goes here, for example ...
-        title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
+        next_data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['episode']
+        title = next_data.get('title')
+        if not title:
+            title = self._og_search_title(webpage)
+        description = next_data.get('description')
+        if not description:
+            description = self._og_search_description(webpage)
+        
+        video_url = next_data.get('m3u8')
+        formats = self._extract_m3u8_formats(
+            video_url, video_id, 'mp4')
+        self._sort_formats(formats)
+
+        
 
         return {
             'id': video_id,
             'title': title,
-            'description': self._og_search_description(webpage),
-            'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
-            # TODO more properties (see youtube_dl/extractor/common.py)
+            'description': description,
+            'formats': formats,
         }
\ No newline at end of file