From 362dbe7469361ea21693939e84acd8a99f7bf759 Mon Sep 17 00:00:00 2001 From: Jeremias Volker Date: Tue, 8 Oct 2019 23:27:38 +0200 Subject: [PATCH 1/3] =?UTF-8?q?Remove=20unused=20function.=20There=20is=20?= =?UTF-8?q?a=20popular=20node=20package=20for=20this=20in=20case=20it?= =?UTF-8?q?=E2=80=99s=20necessary=20in=20the=20future:=20https://www.npmjs?= =?UTF-8?q?.com/package/mkdirp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/downloader.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/downloader.js b/src/downloader.js index e0d157d..1901e5e 100644 --- a/src/downloader.js +++ b/src/downloader.js @@ -10,16 +10,6 @@ const { exec, spawn } = require('child_process'); var debug = require('debug')('openframe:downloader'), artworkDir = '/tmp'; -// unused at present -function _mkdirp(dir) { - var mkdir = 'mkdir -p ' + dir; - exec(mkdir, function(err) { - if (err) { - throw err; - } - }); -} - /** * Download a file using HTTP get. * From a87808c5159bae5bc24952704d0f121fb2a2866f Mon Sep 17 00:00:00 2001 From: Jeremias Volker Date: Wed, 9 Oct 2019 01:55:41 +0200 Subject: [PATCH 2/3] Rewrite of the downloader based on Node request. Clearer error logging and debugging output. #50 --- package.json | 7 ++++ src/downloader.js | 83 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/package.json b/package.json index 93f3159..bc0fed9 100644 --- a/package.json +++ b/package.json @@ -36,9 +36,16 @@ "commander": "^2.8.1", "debug": "^2.2.0", "faye": "^1.1.2", + "http-status": "^1.3.2", + "humanize-duration": "^3.21.0", "inquirer": "^0.11.1", "jsonfile": "^2.2.1", + "mkdirp": "^0.5.1", + "pretty-bytes": "^5.3.0", + "request": "^2.88.0", + "request-progress": "^3.0.0", "shelljs": "^0.7.0", + "single-line-log": "^1.1.2", "swagger-client": "^2.1.9" }, "devDependencies": { diff --git a/src/downloader.js b/src/downloader.js index 1901e5e..6ffc989 100644 --- a/src/downloader.js +++ b/src/downloader.js @@ -5,11 +5,19 @@ */ // Dependencies -const { exec, spawn } = require('child_process'); - -var debug = require('debug')('openframe:downloader'), - artworkDir = '/tmp'; +const fs = require('fs'), + request = require('request'), + progress = require('request-progress'), + mkdirp = require('mkdirp'), + path = require('path'), + logSingleLine = require('single-line-log').stdout, + debug = require('debug')('openframe:downloader'), + status = require('http-status'), + prettyBytes = require('pretty-bytes'), + humanizeDuration = require('humanize-duration'); +const artworkDir = '/tmp'; + /** * Download a file using HTTP get. * @@ -17,31 +25,54 @@ var debug = require('debug')('openframe:downloader'), * @param {String} file_output_name */ function downloadFile(file_url, file_output_name) { - debug('downloading %s', file_url); - return new Promise(function(resolve, reject) { - var file_name = file_output_name, - file_path = artworkDir + '/' + file_name; - - const curl = spawn('curl', ['-L', '-o', file_path, file_url]); - - curl.stdout.on('data', (data) => { - debug(`stdout: ${data}`); - }); + debug('downloading %s', file_url); + return new Promise(function(resolve, reject) { + var file_name = file_output_name, + file_path = artworkDir + '/' + file_name; - curl.stderr.on('data', (data) => { - debug(`stderr: ${data}`); - }); - - curl.on('close', (code) => { - debug(`child process exited with code ${code}`); - if (code !== 0) { - reject('Download failed'); - } else { - resolve(file_path); - } - }); + mkdirp(artworkDir, function (err) { + if (err) { + console.log('Couldn\'t create artwork directory.') + console.error(err) + } }); + progress(request({ + url: file_url, + headers: { + 'User-Agent': 'request' + } + }), { + throttle : 500 + }) + .on('response', function(response) { + // console.log(response.statusCode) + // console.log(response.headers['content-type']) + if (!(/^2/.test('' + response.statusCode))) { // Status Codes other than 2xx + console.log("The artwork is not available.") + console.log("Server responded with this status code:", response.statusCode, status[response.statusCode]); + reject() + } + }) + .on('error', function(err) { + console.log("Error downloading artwork") + console.error(err) + reject() + }) + .on('progress', function (state) { + if (debug.enabled) logSingleLine((state.percent*100).toFixed(2) + '% of ' + prettyBytes(state.size.total)+ ' – ' + humanizeDuration(state.time.remaining * 1000, { round: true }) + ' remaining – ' + (state.speed != null ? prettyBytes(state.speed) : '?') + '/s') + }) + .pipe(fs.createWriteStream(file_path).on("error", function(err) { + console.log("Error saving artwork") + console.log(err) + reject() + })) + .on('finish', function() { + debug('Artwork downloaded') + + return resolve(file_path); + }); + }); } exports.downloadFile = downloadFile; From 0cce2a54e4191902c385ecd59a7626bf600568f3 Mon Sep 17 00:00:00 2001 From: Jeremias Volker Date: Fri, 11 Oct 2019 10:08:41 +0200 Subject: [PATCH 3/3] Abort downloads when next artwork pushed --- src/downloader.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/downloader.js b/src/downloader.js index 6ffc989..74d58a6 100644 --- a/src/downloader.js +++ b/src/downloader.js @@ -17,6 +17,8 @@ const fs = require('fs'), humanizeDuration = require('humanize-duration'); const artworkDir = '/tmp'; + +let artworkRequest, finished = false /** * Download a file using HTTP get. @@ -26,9 +28,10 @@ const artworkDir = '/tmp'; */ function downloadFile(file_url, file_output_name) { debug('downloading %s', file_url); + return new Promise(function(resolve, reject) { var file_name = file_output_name, - file_path = artworkDir + '/' + file_name; + file_path = artworkDir + '/' + file_name; mkdirp(artworkDir, function (err) { if (err) { @@ -37,12 +40,17 @@ function downloadFile(file_url, file_output_name) { } }); - progress(request({ + // debug('finished',finished) + if (artworkRequest && !finished) artworkRequest.abort() + finished = false + artworkRequest = request({ url: file_url, headers: { 'User-Agent': 'request' } - }), { + }) + + progress(artworkRequest, { throttle : 500 }) .on('response', function(response) { @@ -59,6 +67,10 @@ function downloadFile(file_url, file_output_name) { console.error(err) reject() }) + .on('abort', function() { + debug("Aborted downloading artwork") + reject() + }) .on('progress', function (state) { if (debug.enabled) logSingleLine((state.percent*100).toFixed(2) + '% of ' + prettyBytes(state.size.total)+ ' – ' + humanizeDuration(state.time.remaining * 1000, { round: true }) + ' remaining – ' + (state.speed != null ? prettyBytes(state.speed) : '?') + '/s') }) @@ -69,6 +81,7 @@ function downloadFile(file_url, file_output_name) { })) .on('finish', function() { debug('Artwork downloaded') + finished = true return resolve(file_path); });