From 6f7894a242b5cbaacef4b72be4a000bfbffdb398 Mon Sep 17 00:00:00 2001 From: Felipe Andrada Date: Mon, 30 Nov 2020 01:36:50 -0300 Subject: [PATCH 1/6] wait for segment request timeout and auto download retry --- vimeomaster.js | 54 +++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/vimeomaster.js b/vimeomaster.js index d48a00e..06b807c 100644 --- a/vimeomaster.js +++ b/vimeomaster.js @@ -17,20 +17,20 @@ function loadVideo(num, cb) { } const videoData = json.video - .sort((v1, v2) => v1.avg_bitrate - v2.avg_bitrate) - .pop(); + .sort((v1, v2) => v1.avg_bitrate - v2.avg_bitrate) + .pop(); const audioData = json.audio - .sort((a1, a2) => a1.avg_bitrate - a2.avg_bitrate) - .pop(); + .sort((a1, a2) => a1.avg_bitrate - a2.avg_bitrate) + .pop(); const videoBaseUrl = url.resolve( url.resolve(masterUrl, json.base_url), videoData.base_url - ); + ); const audioBaseUrl = url.resolve( url.resolve(masterUrl, json.base_url), audioData.base_url - ); + ); processFile( "video", @@ -56,9 +56,9 @@ function loadVideo(num, cb) { cb(null, num + 1); } - ); + ); } - ); + ); }); } @@ -105,33 +105,37 @@ function combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag "📦", type === "video" ? "📹" : "🎧", `Downloading ${type} segment ${i}/${segmentsUrl.length} of ${filename}` - ); + ); - https - .get(segmentsUrl[i], res => { - res.on("data", d => output.write(d)); + let req = https + .get(segmentsUrl[i], res => { + res.on("data", d => output.write(d)); - res.on("end", () => - combineSegments(type, i + 1, segmentsUrl, output, filename, downloadingFlag, cb) + res.on("end", () => + combineSegments(type, i + 1, segmentsUrl, output, filename, downloadingFlag, cb) ); - }) - .on("error", e => { - cb(e); - }); + }) + .on("error", e => { + cb(e); + }); + req.setTimeout(7000, function () { + console.log("Timeout. Retrying"); + combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb) + }) } function getJson(url, cb) { let data = ""; https - .get(url, res => { - res.on("data", d => (data += d)); + .get(url, res => { + res.on("data", d => (data += d)); - res.on("end", () => cb(null, JSON.parse(data))); - }) - .on("error", e => { - cb(e); - }); + res.on("end", () => cb(null, JSON.parse(data))); + }) + .on("error", e => { + cb(e); + }); } function initJs(n = 0) { From 266a44bbd8376c5ed7238daaf1fd7a37c48f934a Mon Sep 17 00:00:00 2001 From: Felipe Andrada Date: Mon, 30 Nov 2020 01:40:53 -0300 Subject: [PATCH 2/6] undo file identation changes --- vimeomaster.js | 94 +++++++++++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/vimeomaster.js b/vimeomaster.js index 06b807c..d254137 100644 --- a/vimeomaster.js +++ b/vimeomaster.js @@ -9,30 +9,30 @@ function loadVideo(num, cb) { let masterUrl = list[num].url; if (!masterUrl.endsWith("?base64_init=1")) { masterUrl += "?base64_init=1"; - } +} - getJson(masterUrl, (err, json) => { +getJson(masterUrl, (err, json) => { if (err) { return cb(err); - } + } - const videoData = json.video - .sort((v1, v2) => v1.avg_bitrate - v2.avg_bitrate) - .pop(); - const audioData = json.audio - .sort((a1, a2) => a1.avg_bitrate - a2.avg_bitrate) - .pop(); + const videoData = json.video + .sort((v1, v2) => v1.avg_bitrate - v2.avg_bitrate) + .pop(); + const audioData = json.audio + .sort((a1, a2) => a1.avg_bitrate - a2.avg_bitrate) + .pop(); - const videoBaseUrl = url.resolve( + const videoBaseUrl = url.resolve( url.resolve(masterUrl, json.base_url), videoData.base_url ); - const audioBaseUrl = url.resolve( + const audioBaseUrl = url.resolve( url.resolve(masterUrl, json.base_url), audioData.base_url ); - processFile( + processFile( "video", videoBaseUrl, videoData.init_segment, @@ -41,9 +41,9 @@ function loadVideo(num, cb) { err => { if (err) { return cb(err); - } + } - processFile( + processFile( "audio", audioBaseUrl, audioData.init_segment, @@ -52,14 +52,14 @@ function loadVideo(num, cb) { err => { if (err) { return cb(err); - } - - cb(null, num + 1); } - ); + + cb(null, num + 1); } ); - }); + } + ); +}); } function processFile(type, baseUrl, initData, segments, filename, cb) { @@ -68,30 +68,30 @@ function processFile(type, baseUrl, initData, segments, filename, cb) { if(fs.existsSync(downloadingFlag)) { log("⚠️", ` ${filename} - ${type} is incomplete, restarting the download`); - } else if (fs.existsSync(filePath)) { +} else if (fs.existsSync(filePath)) { log("⚠️", ` ${filename} - ${type} already exists`); return cb(); - } else { +} else { fs.writeFileSync(downloadingFlag, ''); - } +} - const segmentsUrl = segments.map(seg => baseUrl + seg.url); +const segmentsUrl = segments.map(seg => baseUrl + seg.url); - const initBuffer = Buffer.from(initData, "base64"); - fs.writeFileSync(filePath, initBuffer); +const initBuffer = Buffer.from(initData, "base64"); +fs.writeFileSync(filePath, initBuffer); - const output = fs.createWriteStream(filePath, { +const output = fs.createWriteStream(filePath, { flags: "a" - }); +}); - combineSegments(type, 0, segmentsUrl, output, filePath, downloadingFlag, err => { +combineSegments(type, 0, segmentsUrl, output, filePath, downloadingFlag, err => { if (err) { log("⚠️", ` ${err}`); - } + } - output.end(); - cb(); - }); + output.end(); + cb(); +}); } function combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb) { @@ -99,29 +99,29 @@ function combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag fs.unlinkSync(downloadingFlag); log("🏁", ` ${filename} - ${type} done`); return cb(); - } +} - log( +log( "📦", type === "video" ? "📹" : "🎧", `Downloading ${type} segment ${i}/${segmentsUrl.length} of ${filename}` ); - let req = https - .get(segmentsUrl[i], res => { +let req = https +.get(segmentsUrl[i], res => { res.on("data", d => output.write(d)); res.on("end", () => combineSegments(type, i + 1, segmentsUrl, output, filename, downloadingFlag, cb) ); - }) - .on("error", e => { +}) +.on("error", e => { cb(e); - }); - req.setTimeout(7000, function () { +}); +req.setTimeout(7000, function () { console.log("Timeout. Retrying"); combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb) - }) +}) } function getJson(url, cb) { @@ -132,10 +132,10 @@ function getJson(url, cb) { res.on("data", d => (data += d)); res.on("end", () => cb(null, JSON.parse(data))); - }) +}) .on("error", e => { cb(e); - }); +}); } function initJs(n = 0) { @@ -145,12 +145,12 @@ function initJs(n = 0) { if (err) { log("⚠️", ` ${err}`); return; - } + } - if (list[num]) { + if (list[num]) { initJs(num); - } - }); + } +}); } initJs(); From 85db8c5885fa1e1b9077353a96b821a5ec936fc9 Mon Sep 17 00:00:00 2001 From: Felipe Andrada Date: Mon, 30 Nov 2020 01:54:14 -0300 Subject: [PATCH 3/6] undo file identation changes --- vimeomaster.js | 145 +++++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 72 deletions(-) diff --git a/vimeomaster.js b/vimeomaster.js index d254137..f95ceea 100644 --- a/vimeomaster.js +++ b/vimeomaster.js @@ -9,30 +9,30 @@ function loadVideo(num, cb) { let masterUrl = list[num].url; if (!masterUrl.endsWith("?base64_init=1")) { masterUrl += "?base64_init=1"; -} - -getJson(masterUrl, (err, json) => { - if (err) { - return cb(err); } - const videoData = json.video - .sort((v1, v2) => v1.avg_bitrate - v2.avg_bitrate) - .pop(); - const audioData = json.audio - .sort((a1, a2) => a1.avg_bitrate - a2.avg_bitrate) - .pop(); + getJson(masterUrl, (err, json) => { + if (err) { + return cb(err); + } - const videoBaseUrl = url.resolve( + const videoData = json.video + .sort((v1, v2) => v1.avg_bitrate - v2.avg_bitrate) + .pop(); + const audioData = json.audio + .sort((a1, a2) => a1.avg_bitrate - a2.avg_bitrate) + .pop(); + + const videoBaseUrl = url.resolve( url.resolve(masterUrl, json.base_url), videoData.base_url - ); - const audioBaseUrl = url.resolve( + ); + const audioBaseUrl = url.resolve( url.resolve(masterUrl, json.base_url), audioData.base_url - ); + ); - processFile( + processFile( "video", videoBaseUrl, videoData.init_segment, @@ -41,9 +41,9 @@ getJson(masterUrl, (err, json) => { err => { if (err) { return cb(err); - } + } - processFile( + processFile( "audio", audioBaseUrl, audioData.init_segment, @@ -52,14 +52,14 @@ getJson(masterUrl, (err, json) => { err => { if (err) { return cb(err); - } + } - cb(null, num + 1); + cb(null, num + 1); + } + ); } - ); - } - ); -}); + ); + }); } function processFile(type, baseUrl, initData, segments, filename, cb) { @@ -68,30 +68,30 @@ function processFile(type, baseUrl, initData, segments, filename, cb) { if(fs.existsSync(downloadingFlag)) { log("⚠️", ` ${filename} - ${type} is incomplete, restarting the download`); -} else if (fs.existsSync(filePath)) { + } else if (fs.existsSync(filePath)) { log("⚠️", ` ${filename} - ${type} already exists`); return cb(); -} else { + } else { fs.writeFileSync(downloadingFlag, ''); -} - -const segmentsUrl = segments.map(seg => baseUrl + seg.url); - -const initBuffer = Buffer.from(initData, "base64"); -fs.writeFileSync(filePath, initBuffer); - -const output = fs.createWriteStream(filePath, { - flags: "a" -}); - -combineSegments(type, 0, segmentsUrl, output, filePath, downloadingFlag, err => { - if (err) { - log("⚠️", ` ${err}`); } - output.end(); - cb(); -}); + const segmentsUrl = segments.map(seg => baseUrl + seg.url); + + const initBuffer = Buffer.from(initData, "base64"); + fs.writeFileSync(filePath, initBuffer); + + const output = fs.createWriteStream(filePath, { + flags: "a" + }); + + combineSegments(type, 0, segmentsUrl, output, filePath, downloadingFlag, err => { + if (err) { + log("⚠️", ` ${err}`); + } + + output.end(); + cb(); + }); } function combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb) { @@ -99,43 +99,44 @@ function combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag fs.unlinkSync(downloadingFlag); log("🏁", ` ${filename} - ${type} done`); return cb(); -} + } -log( + log( "📦", type === "video" ? "📹" : "🎧", `Downloading ${type} segment ${i}/${segmentsUrl.length} of ${filename}` - ); + ); -let req = https -.get(segmentsUrl[i], res => { - res.on("data", d => output.write(d)); + https + .get(segmentsUrl[i], res => { + res.on("data", d => output.write(d)); - res.on("end", () => - combineSegments(type, i + 1, segmentsUrl, output, filename, downloadingFlag, cb) + res.on("end", () => + combineSegments(type, i + 1, segmentsUrl, output, filename, downloadingFlag, cb) ); -}) -.on("error", e => { - cb(e); -}); -req.setTimeout(7000, function () { - console.log("Timeout. Retrying"); - combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb) -}) + }) + .on("error", e => { + cb(e); + }); } function getJson(url, cb) { let data = ""; - https - .get(url, res => { - res.on("data", d => (data += d)); + let req = https + .get(url, res => { + res.on("data", d => (data += d)); - res.on("end", () => cb(null, JSON.parse(data))); -}) - .on("error", e => { - cb(e); -}); + res.on("end", () => cb(null, JSON.parse(data))); + }) + .on("error", e => { + cb(e); + }); + + req.setTimeout(7000, function () { + console.log("Timeout. Retrying"); + combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb) + }); } function initJs(n = 0) { @@ -145,12 +146,12 @@ function initJs(n = 0) { if (err) { log("⚠️", ` ${err}`); return; - } + } - if (list[num]) { + if (list[num]) { initJs(num); - } -}); + } + }); } -initJs(); +initJs(); \ No newline at end of file From 9cefb13322c53a23d3c25a33920bd0aedd4e3b4c Mon Sep 17 00:00:00 2001 From: Felipe A Date: Mon, 30 Nov 2020 01:59:32 -0300 Subject: [PATCH 4/6] refact: add new line at the end of file --- vimeomaster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vimeomaster.js b/vimeomaster.js index f95ceea..cacd782 100644 --- a/vimeomaster.js +++ b/vimeomaster.js @@ -154,4 +154,4 @@ function initJs(n = 0) { }); } -initJs(); \ No newline at end of file +initJs(); From 99138a617e998d7af8b3a4db70b7fe76e9c8c3a5 Mon Sep 17 00:00:00 2001 From: Felipe Andrada Date: Mon, 30 Nov 2020 02:02:44 -0300 Subject: [PATCH 5/6] refact: improve the log on request timeout --- vimeomaster.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vimeomaster.js b/vimeomaster.js index f95ceea..3c299cf 100644 --- a/vimeomaster.js +++ b/vimeomaster.js @@ -134,8 +134,8 @@ function getJson(url, cb) { }); req.setTimeout(7000, function () { - console.log("Timeout. Retrying"); - combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb) + log("⚠️", 'Timeout. Retrying'); + combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb); }); } @@ -154,4 +154,4 @@ function initJs(n = 0) { }); } -initJs(); \ No newline at end of file +initJs(); From 7dc3f70762a206d1a3963db2f3ad970f93e4b1f8 Mon Sep 17 00:00:00 2001 From: Felipe Andrada Date: Mon, 30 Nov 2020 02:14:21 -0300 Subject: [PATCH 6/6] fix: put timeout check in the right location --- vimeomaster.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/vimeomaster.js b/vimeomaster.js index 3c299cf..4ad704b 100644 --- a/vimeomaster.js +++ b/vimeomaster.js @@ -107,7 +107,7 @@ function combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag `Downloading ${type} segment ${i}/${segmentsUrl.length} of ${filename}` ); - https + let req = https .get(segmentsUrl[i], res => { res.on("data", d => output.write(d)); @@ -118,12 +118,17 @@ function combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag .on("error", e => { cb(e); }); + + req.setTimeout(7000, function () { + log("⚠️", 'Timeout. Retrying'); + combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb); + }); } function getJson(url, cb) { let data = ""; - let req = https + https .get(url, res => { res.on("data", d => (data += d)); @@ -132,11 +137,6 @@ function getJson(url, cb) { .on("error", e => { cb(e); }); - - req.setTimeout(7000, function () { - log("⚠️", 'Timeout. Retrying'); - combineSegments(type, i, segmentsUrl, output, filename, downloadingFlag, cb); - }); } function initJs(n = 0) {