From c5e0b5dc3338744f00639ee38cdb8f46b92e9082 Mon Sep 17 00:00:00 2001 From: Gwendal Date: Thu, 15 Nov 2018 21:09:21 +0100 Subject: [PATCH] Look for firefox cookies when no cookies file is provided --- build.gradle | 2 + .../BandcampCollectionDownloader.kt | 139 ++++++++++++++++----- .../kotlin/bandcampcollectiondownloader/Main.kt | 4 +- .../test/BandcampCollectionDownloaderTests.kt | 26 ++++ 4 files changed, 137 insertions(+), 34 deletions(-) diff --git a/build.gradle b/build.gradle index a98e657..c51dca4 100644 --- a/build.gradle +++ b/build.gradle @@ -12,6 +12,8 @@ dependencies { compile 'com.sun.mail:javax.mail:1.6.1' compile 'info.picocli:picocli:3.4.0' compile 'com.google.code.gson:gson:2.8.5' + compile 'org.ini4j:ini4j:0.5.4' + compile 'org.xerial:sqlite-jdbc:3.25.2' testCompile 'org.junit.jupiter:junit-jupiter-api:5.2.0' testRuntime 'org.junit.jupiter:junit-jupiter-engine:5.2.0' } diff --git a/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt b/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt index 1cae265..4c9c0a9 100644 --- a/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt +++ b/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt @@ -3,15 +3,20 @@ package bandcampcollectiondownloader import com.google.gson.Gson import com.google.gson.JsonSyntaxException import com.google.gson.annotations.SerializedName +import org.ini4j.Ini import org.jsoup.HttpStatusException import org.jsoup.Jsoup import org.zeroturnaround.zip.ZipUtil +import java.io.File import java.io.FileOutputStream import java.net.HttpURLConnection import java.net.URL import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.sql.Connection +import java.sql.DriverManager +import java.time.Instant import java.util.* import java.util.regex.Pattern import javax.mail.internet.ContentDisposition @@ -25,7 +30,7 @@ data class ParsedCookie( ) data class ParsedBandcampData( - val digital_items: Array + @Suppress("ArrayInDataClass") val digital_items: Array ) data class DigitalItem( @@ -50,7 +55,7 @@ fun parsedCookiesToMap(parsedCookies: Array): Map if (parsedCookie.nameRaw.isNullOrEmpty()) { throw BandCampDownloaderError("Missing 'Name raw' field in cookie number ${parsedCookies.indexOf(parsedCookie) + 1}.") } - result.put(parsedCookie.nameRaw!!, parsedCookie.contentRaw!!) + result[parsedCookie.nameRaw!!] = parsedCookie.contentRaw!! } return result } @@ -64,24 +69,24 @@ fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = ""): val url = URL(fileURL) val httpConn = url.openConnection() as HttpURLConnection - val responseCode = httpConn.getResponseCode() + val responseCode = httpConn.responseCode // always check HTTP response code first if (responseCode == HttpURLConnection.HTTP_OK) { val disposition = httpConn.getHeaderField("Content-Disposition") val fileName: String = - if (optionalFileName != "") { - optionalFileName - } else if (disposition != null) { - val parsedDisposition = ContentDisposition(disposition) - parsedDisposition.getParameter("filename") - } else { - Paths.get(url.file).fileName.toString() + when { + optionalFileName != "" -> optionalFileName + disposition != null -> { + val parsedDisposition = ContentDisposition(disposition) + parsedDisposition.getParameter("filename") + } + else -> Paths.get(url.file).fileName.toString() } // opens input stream from the HTTP connection - val inputStream = httpConn.getInputStream() + val inputStream = httpConn.inputStream val saveFilePath = saveDir.resolve(fileName) val saveFilePathString = saveFilePath.toAbsolutePath().toString() @@ -106,24 +111,31 @@ fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = ""): } +fun isUnix(): Boolean { + val os = System.getProperty("os.name").toLowerCase() + return os.indexOf("nix") >= 0 || os.indexOf("nux") >= 0 || os.indexOf("aix") > 0 +} /** * Core function called from the main */ -fun downloadAll(cookiesFile: Path, bandcampUser: String, downloadFormat: String, downloadFolder: Path) { - // Parse JSON cookies (obtained with "Cookie Quick Manager" Firefox addon) +fun downloadAll(cookiesFile: Path?, bandcampUser: String, downloadFormat: String, downloadFolder: Path) { val gson = Gson() - if (!Files.exists(cookiesFile)) { - throw BandCampDownloaderError("Cookies file '$cookiesFile' cannot be found.") - } - val jsonData = String(Files.readAllBytes(cookiesFile)) - val parsedCookies = - try { - gson.fromJson(jsonData, Array::class.java) - } catch (e: JsonSyntaxException) { - throw BandCampDownloaderError("Cookies file '$cookiesFile' is not well formed: ${e.message}") + val cookies = + + when { + cookiesFile != null -> { + // Parse JSON cookies (obtained with "Cookie Quick Manager" Firefox addon) + println("Loading provided cookies file: $cookiesFile") + retrieveCookiesFromFile(cookiesFile, gson) + } + isUnix() -> { + // Try to find cookies stored in default firefox profile + println("No provided cookies file, using Firefox cookies.") + retrieveFirefoxCookies() + } + else -> throw BandCampDownloaderError("No available cookies!") } - val cookies = parsedCookiesToMap(parsedCookies) // Get collection page with cookies, hence with download links val doc = try { @@ -152,13 +164,13 @@ fun downloadAll(cookiesFile: Path, bandcampUser: String, downloadFormat: String, val downloadPageJsonParsed = getDataBlobFromDownloadPage(downloadPageURL, cookies, gson) // Extract data from blob - val digitalItem = downloadPageJsonParsed.digital_items.get(0) + val digitalItem = downloadPageJsonParsed.digital_items[0] val albumtitle = digitalItem.title val artist = digitalItem.artist val releaseDate = digitalItem.package_release_date val releaseYear = releaseDate.subSequence(7, 11) val isSingleTrack: Boolean = digitalItem.download_type == "t" - val url = digitalItem.downloads.get(downloadFormat)?.get("url").orEmpty() + val url = digitalItem.downloads[downloadFormat]?.get("url").orEmpty() val artid = digitalItem.art_id // Prepare artist and album folder @@ -171,10 +183,75 @@ fun downloadAll(cookiesFile: Path, bandcampUser: String, downloadFormat: String, } } -class BandCampDownloaderError : Exception { - constructor(s: String) : super(s) +private fun retrieveCookiesFromFile(cookiesFile: Path?, gson: Gson): Map { + if (!Files.exists(cookiesFile)) { + throw BandCampDownloaderError("Cookies file '$cookiesFile' cannot be found.") + } + val jsonData = String(Files.readAllBytes(cookiesFile)) + val parsedCookies = + try { + gson.fromJson(jsonData, Array::class.java) + } catch (e: JsonSyntaxException) { + throw BandCampDownloaderError("Cookies file '$cookiesFile' is not well formed: ${e.message}") + } + return parsedCookiesToMap(parsedCookies) +} + +private fun retrieveFirefoxCookies(): HashMap { + val result = HashMap() + + // Find cookies file path + val homeDir = System.getenv()["HOME"] + val firefoxConfDirPath = "$homeDir/.mozilla/firefox" + val profilesListPath = "$firefoxConfDirPath/profiles.ini" + val profilesListFile = File(profilesListPath) + if (!profilesListFile.exists()) { + throw BandCampDownloaderError("No Firefox profiles.ini file could be found!") + } + val ini = Ini(profilesListFile) + val default = "Default" + val defaultProfileSection = ini.keys.find { + ini[it] != null + && ini[it]!!.containsKey(default) + && ini[it]!![default] == "1" + } + val defaultProfilePath = firefoxConfDirPath + "/" + ini.get(defaultProfileSection, "Path") + val cookiesFilePath = "$defaultProfilePath/cookies.sqlite" + + // Copy cookies file as tmp file + val tmpFolder = Files.createTempDirectory("bandcampCollectionDownloader") + val copiedCookiesPath = Files.copy(Paths.get(cookiesFilePath), tmpFolder.resolve("cookies.json")) + copiedCookiesPath.toFile().deleteOnExit() + + // Start reading firefox's cookies.sqlite + var connection: Connection? = null + try { + // create a database connection + connection = DriverManager.getConnection("jdbc:sqlite:$copiedCookiesPath") + val statement = connection!!.createStatement() + statement.queryTimeout = 30 // set timeout to 30 sec. + val rs = statement.executeQuery("select * from moz_cookies where baseDomain = 'bandcamp.com'") + // For each resulting row + while (rs.next()) { + // Extract data from row + val name = rs.getString("name") + val value = rs.getString("value") + val expiry = rs.getString("expiry").toLong() + + // We only keep cookies that have not expired yet + val now = Instant.now().epochSecond + val difference = expiry - now + if (difference > 0) + result[name] = value + } + } finally { + connection?.close() + } + return result } +class BandCampDownloaderError(s: String) : Exception(s) + fun downloadAlbum(artistFolderPath: Path?, albumFolderPath: Path, albumtitle: String, url: String, cookies: Map, gson: Gson, isSingleTrack: Boolean, artid: String) { // If the artist folder does not exist, we create it if (!Files.exists(artistFolderPath)) { @@ -228,8 +305,7 @@ fun getDataBlobFromDownloadPage(downloadPageURL: String?, cookies: Map, gson: Gson, albumFolderPath: Path): Path { @@ -247,7 +323,7 @@ fun prepareDownload(albumtitle: String, url: String, cookies: Map) { // Else, parse arguments and run else { val bandcampUser = parsedArgs.bandcampUser - val cookiesFile = parsedArgs.pathToCookiesFile!! + val cookiesFile = parsedArgs.pathToCookiesFile val downloadFormat = parsedArgs.audioFormat val downloadFolder = parsedArgs.pathToDownloadFolder try { diff --git a/src/test/kotlin/bandcampcollectiodownloader/test/BandcampCollectionDownloaderTests.kt b/src/test/kotlin/bandcampcollectiodownloader/test/BandcampCollectionDownloaderTests.kt index 2168492..f61ac92 100644 --- a/src/test/kotlin/bandcampcollectiodownloader/test/BandcampCollectionDownloaderTests.kt +++ b/src/test/kotlin/bandcampcollectiodownloader/test/BandcampCollectionDownloaderTests.kt @@ -5,6 +5,7 @@ import bandcampcollectiondownloader.downloadAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import java.nio.file.Paths +import java.util.* /** * Note: bli is a valid bandcamp user (completely randomly chosen), @@ -54,4 +55,29 @@ class BandcampCollectionDownloaderTests { } } + @Test + fun testErrorNoCookiesAtAll() { + addToEnv("HOME", "NOPE") + assertThrows { + downloadAll(null, "bli", "bli", Paths.get("bli")) + } + } + + + @Throws(Exception::class) + fun addToEnv(key: String, value: String) { + val classes = Collections::class.java!!.declaredClasses + val env = System.getenv() + for (cl in classes) { + if ("java.util.Collections\$UnmodifiableMap" == cl.name) { + val field = cl.getDeclaredField("m") + field.isAccessible = true + val obj = field.get(env) + val map = obj as MutableMap + map[key] = value + } + } + } + + } \ No newline at end of file