From a7202ebe83b46b8a6188b30a915c59dc724b4275 Mon Sep 17 00:00:00 2001 From: Gwendal Date: Fri, 4 Jan 2019 14:43:21 +0100 Subject: [PATCH] Add timeout option for HTTP connections --- .../BandcampCollectionDownloader.kt | 25 ++++++++++++---------- .../kotlin/bandcampcollectiondownloader/Main.kt | 8 +++++-- .../kotlin/bandcampcollectiondownloader/Util.kt | 7 +++++- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt b/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt index bf7e583..eeec17b 100644 --- a/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt +++ b/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt @@ -6,6 +6,7 @@ import org.jsoup.Jsoup import org.zeroturnaround.zip.ZipUtil import retrieveCookiesFromFile import retrieveFirefoxCookies +import java.lang.Thread.sleep import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -34,7 +35,7 @@ data class ParsedStatDownload( /** * Core function called from the main */ -fun downloadAll(cookiesFile: Path?, bandcampUser: String, downloadFormat: String, downloadFolder: Path, retries: Int) { +fun downloadAll(cookiesFile: Path?, bandcampUser: String, downloadFormat: String, downloadFolder: Path, retries: Int, timeout: Int) { val gson = Gson() val cookies = @@ -51,6 +52,7 @@ fun downloadAll(cookiesFile: Path?, bandcampUser: String, downloadFormat: String // Get collection page with cookies, hence with download links val doc = try { Jsoup.connect("https://bandcamp.com/$bandcampUser") + .timeout(timeout) .cookies(cookies) .get() } catch (e: HttpStatusException) { @@ -72,7 +74,7 @@ fun downloadAll(cookiesFile: Path?, bandcampUser: String, downloadFormat: String // For each download page for (item in collection) { val downloadPageURL = item.attr("href") - val downloadPageJsonParsed = getDataBlobFromDownloadPage(downloadPageURL, cookies, gson) + val downloadPageJsonParsed = getDataBlobFromDownloadPage(downloadPageURL, cookies, gson, timeout) // Extract data from blob val digitalItem = downloadPageJsonParsed.digital_items[0] @@ -98,9 +100,10 @@ fun downloadAll(cookiesFile: Path?, bandcampUser: String, downloadFormat: String for (i in 1..attempts) { if (i > 1) { println("Retrying download (${i - 1}/$retries).") + sleep(1000) } try { - downloadAlbum(artistFolderPath, albumFolderPath, albumtitle, url, cookies, gson, isSingleTrack, artid) + downloadAlbum(artistFolderPath, albumFolderPath, albumtitle, url, cookies, gson, isSingleTrack, artid, timeout) break } catch (e: Throwable) { println("""Error while downloading: "${e.javaClass.name}: ${e.message}".""") @@ -115,7 +118,7 @@ fun downloadAll(cookiesFile: Path?, bandcampUser: String, downloadFormat: String class BandCampDownloaderError(s: String) : Exception(s) -fun downloadAlbum(artistFolderPath: Path?, albumFolderPath: Path, albumtitle: String, url: String, cookies: Map, gson: Gson, isSingleTrack: Boolean, artid: String) { +fun downloadAlbum(artistFolderPath: Path?, albumFolderPath: Path, albumtitle: String, url: String, cookies: Map, gson: Gson, isSingleTrack: Boolean, artid: String, timeout: Int) { // If the artist folder does not exist, we create it if (!Files.exists(artistFolderPath)) { Files.createDirectories(artistFolderPath) @@ -130,7 +133,7 @@ fun downloadAlbum(artistFolderPath: Path?, albumFolderPath: Path, albumtitle: St val amountFiles = albumFolderPath.toFile().listFiles().size if (amountFiles < 2) { - val outputFilePath: Path = prepareDownload(albumtitle, url, cookies, gson, albumFolderPath) + val outputFilePath: Path = prepareDownload(albumtitle, url, cookies, gson, albumFolderPath, timeout) // If this is a zip, we unzip if (!isSingleTrack) { @@ -148,7 +151,7 @@ fun downloadAlbum(artistFolderPath: Path?, albumFolderPath: Path, albumtitle: St else { val coverURL = "https://f4.bcbits.com/img/a${artid}_10" println("Downloading cover ($coverURL)...") - downloadFile(coverURL, albumFolderPath, "cover.jpg") + downloadFile(coverURL, albumFolderPath, "cover.jpg", timeout) } println("done.") @@ -158,20 +161,20 @@ fun downloadAlbum(artistFolderPath: Path?, albumFolderPath: Path, albumtitle: St } } -fun getDataBlobFromDownloadPage(downloadPageURL: String?, cookies: Map, gson: Gson): ParsedBandcampData { +fun getDataBlobFromDownloadPage(downloadPageURL: String?, cookies: Map, gson: Gson, timeout: Int): ParsedBandcampData { println("Analyzing download page $downloadPageURL") // Get page content val downloadPage = Jsoup.connect(downloadPageURL) .cookies(cookies) - .timeout(100000).get() + .timeout(timeout).get() // Get data blob val downloadPageJson = downloadPage.select("#pagedata").attr("data-blob") return gson.fromJson(downloadPageJson, ParsedBandcampData::class.java) } -fun prepareDownload(albumtitle: String, url: String, cookies: Map, gson: Gson, albumFolderPath: Path): Path { +fun prepareDownload(albumtitle: String, url: String, cookies: Map, gson: Gson, albumFolderPath: Path, timeout: Int): Path { println("Preparing download of $albumtitle ($url)...") val random = Random() @@ -185,7 +188,7 @@ fun prepareDownload(albumtitle: String, url: String, cookies: Map) { val downloadFormat = parsedArgs.audioFormat val downloadFolder = parsedArgs.pathToDownloadFolder val retries = parsedArgs.retries + val timeout = parsedArgs.timeout try { - downloadAll(cookiesFile, bandcampUser, downloadFormat, downloadFolder, retries) + downloadAll(cookiesFile, bandcampUser, downloadFormat, downloadFolder, retries, timeout) } catch (e: BandCampDownloaderError) { System.err.println("ERROR: ${e.message}") } diff --git a/src/main/kotlin/bandcampcollectiondownloader/Util.kt b/src/main/kotlin/bandcampcollectiondownloader/Util.kt index f29a236..767c8b3 100644 --- a/src/main/kotlin/bandcampcollectiondownloader/Util.kt +++ b/src/main/kotlin/bandcampcollectiondownloader/Util.kt @@ -14,10 +14,12 @@ const val BUFFER_SIZE = 4096 /** * From http://www.codejava.net/java-se/networking/use-httpurlconnection-to-download-file-from-an-http-url */ -fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = ""): Path { +fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = "", timeout: Int): Path { val url = URL(fileURL) val httpConn = url.openConnection() as HttpURLConnection + httpConn.connectTimeout = timeout + httpConn.readTimeout = timeout val responseCode = httpConn.responseCode // always check HTTP response code first @@ -47,10 +49,13 @@ fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = ""): var bytesRead = inputStream.read(buffer) var total = 0 while (bytesRead != -1) { + // Print progress val percent = total.toDouble() / fileSize.toDouble() * 100 val formatter = DecimalFormat("#0.00") val percentString = formatter.format(percent) System.out.print("Progress: $percentString % ($total / $fileSize) \r") + + // Download chunk outputStream.write(buffer, 0, bytesRead) bytesRead = inputStream.read(buffer) total += bytesRead