diff --git a/build.gradle b/build.gradle index 6dacd35..a98e657 100644 --- a/build.gradle +++ b/build.gradle @@ -12,6 +12,8 @@ dependencies { compile 'com.sun.mail:javax.mail:1.6.1' compile 'info.picocli:picocli:3.4.0' compile 'com.google.code.gson:gson:2.8.5' + testCompile 'org.junit.jupiter:junit-jupiter-api:5.2.0' + testRuntime 'org.junit.jupiter:junit-jupiter-engine:5.2.0' } compileKotlin { kotlinOptions { @@ -27,7 +29,7 @@ compileTestKotlin { //create a single Jar with all dependencies task fatJar(type: Jar) { manifest { - attributes 'Main-Class': 'bandcampcollectiondownloader.BandcampCollectionDownloaderKt' + attributes 'Main-Class': 'bandcampcollectiondownloader.MainKt' } baseName = project.name from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } diff --git a/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt b/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt index 9e53916..8e6829a 100644 --- a/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt +++ b/src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt @@ -4,7 +4,6 @@ import com.google.gson.Gson import com.google.gson.annotations.SerializedName import org.jsoup.Jsoup import org.zeroturnaround.zip.ZipUtil -import picocli.CommandLine import java.io.FileOutputStream import java.net.HttpURLConnection import java.net.URL @@ -100,54 +99,10 @@ fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = ""): } -data class Args( - @CommandLine.Parameters(arity = "1..1", - description = arrayOf("The bandcamp user account from which all albums must be downloaded.")) - var bandcampUser: String = "", - - @CommandLine.Option(names = arrayOf("--cookies-file", "-c"), required = true, - description = arrayOf("A JSON file with valid bandcamp credential cookies.", - """"Cookie Quick Manager" can be used to obtain this file after logging into bandcamp.""", - "(visit https://addons.mozilla.org/en-US/firefox/addon/cookie-quick-manager/).")) - var pathToCookiesFile: String? = null, - - @CommandLine.Option(names = arrayOf("--audio-format", "-f"), required = false, - description = arrayOf("The chosen audio format of the files to download (default: \${DEFAULT-VALUE}).", - "Possible values: flac, wav, aac-hi, mp3-320, aiff-lossless, vorbis, mp3-v0, alac.")) - var audioFormat: String = "vorbis", - - @CommandLine.Option(names = arrayOf("--download-folder", "-d"), required = false, - description = arrayOf("The folder in which downloaded albums must be extracted.", - "The following structure is considered: // - .", - "(default: current folder)")) - var pathToDownloadFolder: Path = Paths.get("."), - - @CommandLine.Option(names = arrayOf("-h", "--help"), usageHelp = true, description = arrayOf("Display this help message.")) - var help: Boolean = false - -) - - -fun main(args: Array) { - - // Parsing args - System.setProperty("picocli.usage.width", "120") - val parsedArgs: Args = - try { - CommandLine.populateCommand(Args(), *args) - } catch (e: CommandLine.MissingParameterException) { - CommandLine.usage(Args(), System.out) - System.err.println(e.message) - return - } - if (parsedArgs.help) { - CommandLine.usage(Args(), System.out) - } - val bandcampUser = parsedArgs.bandcampUser - val cookiesFile = parsedArgs.pathToCookiesFile - val downloadFormat = parsedArgs.audioFormat - val downloadFolder = parsedArgs.pathToDownloadFolder - +/** + * Core function called from the main + */ +fun downloadAll(cookiesFile: String, bandcampUser: String, downloadFormat: String, downloadFolder: Path) { // Parse JSON cookies (obtained with "Cookie Quick Manager" Firefox addon) val gson = Gson() val jsonData = String(Files.readAllBytes(Paths.get(cookiesFile))) @@ -169,16 +124,7 @@ fun main(args: Array) { // For each download page for (item in collection) { val downloadPageURL = item.attr("href") - println("Analyzing download page $downloadPageURL") - - // Get page content - val downloadPage = Jsoup.connect(downloadPageURL) - .cookies(cookies) - .timeout(100000).get() - - // Get data blob - val downloadPageJson = downloadPage.select("#pagedata").attr("data-blob") - val downloadPageJsonParsed = gson.fromJson(downloadPageJson, ParsedBandcampData::class.java) + val downloadPageJsonParsed = getDataBlobFromDownloadPage(downloadPageURL, cookies, gson) // Extract data from blob val digitalItem = downloadPageJsonParsed.digital_items.get(0) @@ -195,79 +141,100 @@ fun main(args: Array) { val artistFolderPath = Paths.get("$downloadFolder").resolve(artist) val albumFolderPath = artistFolderPath.resolve(albumFolderName) - // If the artist folder does not exist, we create it - if (!Files.exists(artistFolderPath)) { - Files.createDirectory(artistFolderPath) - } - - // If the album folder does not exist, we create it - if (!Files.exists(albumFolderPath)) { - Files.createDirectory(albumFolderPath) - } + downloadAlbum(artistFolderPath, albumFolderPath, albumtitle, url, cookies, gson, isSingleTrack, artid) - // If the folder is empty, or if it only contains the zip.part file, we proceed - val amountFiles = albumFolderPath.toFile().listFiles().size - if (amountFiles < 2) { + } +} - println("Preparing download of $albumtitle ($url)...") +fun downloadAlbum(artistFolderPath: Path?, albumFolderPath: Path, albumtitle: String, url: String, cookies: Map, gson: Gson, isSingleTrack: Boolean, artid: String) { + // If the artist folder does not exist, we create it + if (!Files.exists(artistFolderPath)) { + Files.createDirectory(artistFolderPath) + } - val random = Random() + // If the album folder does not exist, we create it + if (!Files.exists(albumFolderPath)) { + Files.createDirectory(albumFolderPath) + } - // Construct statdownload request URL - val statdownloadURL: String = url - .replace("/download/", "/statdownload/") - .replace("http", "https") + "&.vrs=1" + "&.rand=" + random.nextInt() + // If the folder is empty, or if it only contains the zip.part file, we proceed + val amountFiles = albumFolderPath.toFile().listFiles().size + if (amountFiles < 2) { - // Get statdownload JSON - println("Getting download link ($statdownloadURL)") - val statedownloadUglyBody: String = Jsoup.connect(statdownloadURL) - .cookies(cookies) - .timeout(100000) - .get().body().select("body").get(0).text().toString() + val outputFilePath: Path = prepareDownload(albumtitle, url, cookies, gson, albumFolderPath) - val prefixPattern = Pattern.compile("""if\s*\(\s*window\.Downloads\s*\)\s*\{\s*Downloads\.statResult\s*\(\s*""") - val suffixPattern = Pattern.compile("""\s*\)\s*};""") - val statdownloadJSON: String = - prefixPattern.matcher( - suffixPattern.matcher(statedownloadUglyBody) - .replaceAll("") - ).replaceAll("") + // If this is a zip, we unzip + if (!isSingleTrack) { - // Parse statdownload JSON and get real download URL, and retrieve url - val statdownloadParsed: ParsedStatDownload = gson.fromJson(statdownloadJSON, ParsedStatDownload::class.java) - val realDownloadURL = statdownloadParsed.download_url + // Unzip + try { + ZipUtil.unpack(outputFilePath.toFile(), albumFolderPath.toFile()) + } finally { + // Delete zip + Files.delete(outputFilePath) + } + } - println("Downloading $albumtitle ($realDownloadURL)") + // Else if this is a single track, we just fetch the cover + else { + val coverURL = "https://f4.bcbits.com/img/a${artid}_10" + println("Downloading cover ($coverURL)...") + downloadFile(coverURL, albumFolderPath, "cover.jpg") + } - // Download content - val outputFilePath: Path = downloadFile(realDownloadURL, albumFolderPath) + println("done.") - // If this is a zip, we unzip - if (!isSingleTrack) { + } else { + println("Album $albumtitle already done, skipping") + } +} - // Unzip - try { - ZipUtil.unpack(outputFilePath.toFile(), albumFolderPath.toFile()) - } finally { - // Delete zip - Files.delete(outputFilePath) - } - } +fun getDataBlobFromDownloadPage(downloadPageURL: String?, cookies: Map, gson: Gson): ParsedBandcampData { + println("Analyzing download page $downloadPageURL") - // Else if this is a single track, we just fetch the cover - else { - val coverURL = "https://f4.bcbits.com/img/a${artid}_10" - println("Downloading cover ($coverURL)...") - downloadFile(coverURL, albumFolderPath, "cover.jpg") - } + // Get page content + val downloadPage = Jsoup.connect(downloadPageURL) + .cookies(cookies) + .timeout(100000).get() - println("done.") + // Get data blob + val downloadPageJson = downloadPage.select("#pagedata").attr("data-blob") + val downloadPageJsonParsed = gson.fromJson(downloadPageJson, ParsedBandcampData::class.java) + return downloadPageJsonParsed +} - } else { - println("Album $albumtitle already done, skipping") - } +fun prepareDownload(albumtitle: String, url: String, cookies: Map, gson: Gson, albumFolderPath: Path): Path { + println("Preparing download of $albumtitle ($url)...") - } + val random = Random() + // Construct statdownload request URL + val statdownloadURL: String = url + .replace("/download/", "/statdownload/") + .replace("http", "https") + "&.vrs=1" + "&.rand=" + random.nextInt() + // Get statdownload JSON + println("Getting download link ($statdownloadURL)") + val statedownloadUglyBody: String = Jsoup.connect(statdownloadURL) + .cookies(cookies) + .timeout(100000) + .get().body().select("body").get(0).text().toString() + + val prefixPattern = Pattern.compile("""if\s*\(\s*window\.Downloads\s*\)\s*\{\s*Downloads\.statResult\s*\(\s*""") + val suffixPattern = Pattern.compile("""\s*\)\s*};""") + val statdownloadJSON: String = + prefixPattern.matcher( + suffixPattern.matcher(statedownloadUglyBody) + .replaceAll("") + ).replaceAll("") + + // Parse statdownload JSON and get real download URL, and retrieve url + val statdownloadParsed: ParsedStatDownload = gson.fromJson(statdownloadJSON, ParsedStatDownload::class.java) + val realDownloadURL = statdownloadParsed.download_url + + println("Downloading $albumtitle ($realDownloadURL)") + + // Download content + val outputFilePath: Path = downloadFile(realDownloadURL, albumFolderPath) + return outputFilePath } \ No newline at end of file diff --git a/src/main/kotlin/bandcampcollectiondownloader/Main.kt b/src/main/kotlin/bandcampcollectiondownloader/Main.kt new file mode 100644 index 0000000..afa6987 --- /dev/null +++ b/src/main/kotlin/bandcampcollectiondownloader/Main.kt @@ -0,0 +1,58 @@ +package bandcampcollectiondownloader + +import picocli.CommandLine +import java.nio.file.Path +import java.nio.file.Paths + + +data class Args( + @CommandLine.Parameters(arity = "1..1", + description = arrayOf("The bandcamp user account from which all albums must be downloaded.")) + var bandcampUser: String = "", + + @CommandLine.Option(names = arrayOf("--cookies-file", "-c"), required = true, + description = arrayOf("A JSON file with valid bandcamp credential cookies.", + """"Cookie Quick Manager" can be used to obtain this file after logging into bandcamp.""", + "(visit https://addons.mozilla.org/en-US/firefox/addon/cookie-quick-manager/).")) + var pathToCookiesFile: String? = null, + + @CommandLine.Option(names = arrayOf("--audio-format", "-f"), required = false, + description = arrayOf("The chosen audio format of the files to download (default: \${DEFAULT-VALUE}).", + "Possible values: flac, wav, aac-hi, mp3-320, aiff-lossless, vorbis, mp3-v0, alac.")) + var audioFormat: String = "vorbis", + + @CommandLine.Option(names = arrayOf("--download-folder", "-d"), required = false, + description = arrayOf("The folder in which downloaded albums must be extracted.", + "The following structure is considered: // - .", + "(default: current folder)")) + var pathToDownloadFolder: Path = Paths.get("."), + + @CommandLine.Option(names = arrayOf("-h", "--help"), usageHelp = true, description = arrayOf("Display this help message.")) + var help: Boolean = false + +) + + +fun main(args: Array) { + + // Parsing args + System.setProperty("picocli.usage.width", "120") + val parsedArgs: Args = + try { + CommandLine.populateCommand(Args(), *args) + } catch (e: CommandLine.MissingParameterException) { + CommandLine.usage(Args(), System.out) + System.err.println(e.message) + return + } + if (parsedArgs.help) { + CommandLine.usage(Args(), System.out) + } else { + val bandcampUser = parsedArgs.bandcampUser + val cookiesFile = parsedArgs.pathToCookiesFile!! + val downloadFormat = parsedArgs.audioFormat + val downloadFolder = parsedArgs.pathToDownloadFolder + downloadAll(cookiesFile, bandcampUser, downloadFormat, downloadFolder) + } + +}