Fixed donwloader to work with direct cookie. Cleaned some files.

.gitignore

@@ -1,5 +1,4 @@
-login.json
-release.json
+config.json
 cookies.txt
 mcat_dl*
 .vscode/*

Makefile

@@ -1,4 +1,9 @@
+srcs += main.cpp
+srcs += curl_dl.cpp
+srcs += monstercat_dl.cpp
+srcs += common.cpp
+
 all:
-	g++ main.cpp -o mcat_dl
+	g++ ${srcs} -lcurl -o mcat_dl
 
 .PHONY: all

README

@@ -1,44 +0,0 @@
---- Usage ---
-There must be a file named "login.json" in the working directory with your monstercat credentials.
-See example_login.json for the template.
-
---- JSON Library Source ---
-https://github.com/nlohmann/json
-Release - 3.10.5
-Commit - 4f8fba14066156b73f1189a2b8bd568bde5284c5
-
---- id3edit ---
-https://github.com/rstemmer/id3edit
-For MP3 tagging
-id3edit
-	--set-name "Title"
-	--set-album "Album"
-	--set-artist "Artist"
-	--set-track "Track Number"
-	--set-artwork "/path/to/cover"
-	file.mp3
-
---- metaflac ---
-https://xiph.org/flac/download.html
-For FLAC tagging
-metaflac
-	// Common
-	--preserve-modtime
-	--no-utf8-convert
-
-	// First Step - Remove
-	--remove --block-type=PICTURE
-	--remove-tag=TITLE
-	--remove-tag=ARTIST
-	--remove-tag=ALBUM
-	--remove-tag=TRACKNUMBER
-
-	// Second Step - Add
-	--import-picture-from=3|image/jpeg|||"/path/to/cover"
-	"--set-tag=TITLE=..."
-	"--set-tag=ARTIST=..."
-	"--set-tag=ALBUM=..."
-	"--set-tag=TRACKNUMBER=..."
-	--dont-use-padding
-
-	file.flac

README.md

@@ -0,0 +1,109 @@
+# Prerequisites
+### Ubuntu
+`apt install build-essential libcurl4-gnutls-dev imagemagick eyeD3 flac`
+
+### Other OS
+* make
+* g++
+* libcurl
+* imagemagick (convert executable)
+* eyeD3 executable
+* metaflac executable
+
+# Usage
+There must be a file named "config.json" in the working directory with your monstercat credentials and other configurations.
+See config_example.json for the template.
+
+Invoke `make` to compile. Requires libcurl.
+
+Login via browser and export `cid` cookie to cookies.txt (Netscape format)
+
+Invoke `mcat_dl` with catalog ID as extra arguments to download.
+
+# Download File Structure
+### Release Folder
+**download_path/Type/YYYY-MM-DD - CatalogID - Artist - Title**
+* download_path - Set in config.json
+* Type - Album/EP/Single
+* YYYY-MM-DD - Release Date in ISO format for easy sorting
+* Artist/Title - removed featuring artists from "Artist" and added to "Title"
+
+### Cover
+* Cover(.jpg/.png) - Full resolution from Monstercat
+* Cover_small.jpg - 750x750 used to tag files
+
+### MP3 and FLAC folders
+Separate folders if more than 1 track otherwise put in main folder.
+
+### Extended folder
+This is where extended mixes are put into. They are in their original format.
+
+### File Naming
+* Title.(mp3/flac) - Used when only 1 track
+
+* Number - Title.(mp3/flac) - Used when track artist **matches** release artist
+* Number - Artist - Title.(mp3/flac) - Used when track artist **does not match** release artist
+
+# JSON Library Source
+https://github.com/nlohmann/json
+* Release - 3.10.5
+* Commit - 4f8fba14066156b73f1189a2b8bd568bde5284c5
+
+# eyeD3
+https://eyed3.readthedocs.io/en/latest/
+* For MP3 tagging
+```
+	// Common
+	--encoding utf8
+	--preserve-file-times
+
+	// First Step - Remove Images
+	--remove-all-images
+
+	// Second Step - Change what is needed
+	--add-image "...":FRONT_COVER
+	--artist "..."
+	--album "..."
+	--title "..."
+	--track ...
+
+```
+
+# metaflac
+https://xiph.org/flac/download.html
+* For FLAC tagging
+```
+metaflac
+	// Common
+	--preserve-modtime
+	--no-utf8-convert
+
+	// First Step - Remove Pictures
+	--remove --block-type=PICTURE
+
+	// Second Step - Remove Tags
+	--remove-tag=TITLE
+	--remove-tag=ARTIST
+	--remove-tag=ALBUM
+	--remove-tag=TRACKNUMBER
+
+	// Third Step - Add
+	"--import-picture-from=3|image/jpeg|||/path/to/cover"
+	"--set-tag=TITLE=..."
+	"--set-tag=ARTIST=..."
+	"--set-tag=ALBUM=..."
+	"--set-tag=TRACKNUMBER=..."
+	--dont-use-padding
+
+	file.flac
+```
+
+# imagemagick
+https://imagemagick.org/script/download.php
+* For Image Resizing
+```
+convert
+	Cover
+	-resize 750x750
+	Cover_small.jpg
+```

TODO

@@ -1,2 +0,0 @@
-Add Cover Resize via lib or command ???
-Implement main

TODO.md

@@ -0,0 +1 @@
+* Add API Rate Limit

common.cpp

@@ -0,0 +1,203 @@
+#include "common.h"
+
+#include <fstream>
+#include <set>
+
+#include <stdlib.h>
+#include <sys/stat.h>
+
+using std::endl;
+using std::ifstream;
+using std::istream;
+using std::ostream;
+using std::set;
+using std::string;
+using std::to_string;
+using nlohmann::json;
+
+bool read_config(json& cfg, ostream* log)
+{
+	ifstream cfg_file("config.json");
+
+	if (!cfg_file.is_open())
+	{
+		if (nullptr != log)
+		{
+			*log << "Could not open config.json" << endl;
+		}
+		return false;
+	}
+
+	// Parse with comments
+	try
+	{
+		cfg = json::parse(cfg_file, nullptr, true, true);
+	}
+	catch (const std::exception &e)
+	{
+		if (nullptr != log)
+		{
+			*log << e.what() << endl;
+		}
+		return false;
+	}
+
+	return true;
+}
+
+bool path_exists(const string& path)
+{
+	struct stat info;
+	int error;
+
+	error = stat(path.c_str(), &info);
+	if (0 != error)
+	{
+		return false;
+	}
+
+	if (S_ISDIR(info.st_mode))
+	{
+		return true;
+	}
+
+	return false;
+}
+
+string build_fname(const string& main_path, const string& folder, const string& fname)
+{
+	string path_to_file;
+
+	path_to_file = main_path;
+	if (!path_to_file.empty() && path_to_file[path_to_file.size() - 1] != '/')
+	{
+		path_to_file += '/';
+	}
+
+	path_to_file += folder;
+	if (!path_to_file.empty() && path_to_file[path_to_file.size() - 1] != '/')
+	{
+		path_to_file += '/';
+	}
+
+	path_to_file += fname;
+
+	return path_to_file;
+}
+
+string i_to_str(int i, int size, char fill)
+{
+	string str;
+
+	str = to_string(i);
+	str.insert(str.begin(), size - str.size(), fill);
+
+	return str;
+}
+
+void json_extract(const json& j, const string& key, string& out)
+{
+	if (j.contains(key) && j[key].is_string())
+	{
+		out = j[key];
+	}
+}
+
+void json_extract(const json& j, const string& key, uint16_t& out)
+{
+	if (j.contains(key) && j[key].is_number_unsigned())
+	{
+		out = j[key];
+	}
+}
+
+void json_extract(const json& j, const string& key, bool& out)
+{
+	if (j.contains(key) && j[key].is_boolean())
+	{
+		out = j[key];
+	}
+}
+
+bool ensure_folder(const string& main_path, const string& folder)
+{
+	string full_path;
+	int error;
+
+	if (!path_exists(main_path))
+	{
+		error = mkdir(main_path.c_str(), 0775);
+		if (0 != error)
+		{
+			return false;
+		}
+	}
+
+	full_path = build_fname(main_path, folder, "");
+	if (path_exists(full_path))
+	{
+		return true;
+	}
+
+	error = mkdir(full_path.c_str(), 0775);
+	if (0 != error)
+	{
+		return false;
+	}
+
+	return true;
+}
+
+string trim_whitespace(const string& s)
+{
+	const string whitespace = " \t\r\n\f\v";
+	string result(s);
+	size_t pos;
+
+	pos = result.find_first_not_of(whitespace);
+	result.erase(0, pos);
+	pos = result.find_last_not_of(whitespace);
+	result.erase(pos + 1);
+
+	return result;
+}
+
+string clean_filename(const string& s)
+{
+	string result = s;
+
+	set<char> illegal;
+
+	illegal.insert('/');
+    illegal.insert('\\');
+    illegal.insert('<');
+    illegal.insert('>');
+    illegal.insert(':');
+    illegal.insert('"');
+    illegal.insert('|');
+    illegal.insert('?');
+    illegal.insert('*');
+
+	for (char& c : result)
+	{
+		if (0 != illegal.count(c))
+		{
+			c = ';';
+		}
+	}
+
+	return result;
+}
+
+bool exec_cmd(const string& cmd)
+{
+	int error;
+
+	error = system(cmd.c_str());
+	if (0 != error)
+	{
+		return false;
+	}
+
+	return true;
+}

common.h

@@ -0,0 +1,24 @@
+#ifndef COMMON_H_
+#define COMMON_H_
+
+#include <iostream>
+#include <set>
+#include <string>
+#include "json.hpp"
+
+bool read_config(nlohmann::json& cfg, std::ostream* log = &std::cout);
+bool path_exists(const std::string& path);
+std::string build_fname(const std::string& main_path, const std::string& folder,
+	const std::string& fname);
+std::string i_to_str(int i, int size, char fill = '0');
+void json_extract(const nlohmann::json& j, const std::string& key,
+	std::string& out);
+void json_extract(const nlohmann::json& j, const std::string& key,
+	uint16_t& out);
+void json_extract(const nlohmann::json& j, const std::string& key, bool& out);
+bool ensure_folder(const std::string& main_path, const std::string& folder);
+std::string trim_whitespace(const std::string& s);
+std::string clean_filename(const std::string& s);
+bool exec_cmd(const std::string& cmd);
+
+#endif // COMMON_H_

config_example.json

@@ -0,0 +1,8 @@
+{
+	"Email": "user@example.com",
+	"Pass": "password",
+	"download_path": "/path/to/monstercat",
+	"convert_exec": "convert",
+	"eyed3_exec": "eyeD3",
+	"metaflac_exec": "metaflac"
+}

curl_dl.cpp

@@ -0,0 +1,437 @@
+#include "curl_dl.h"
+
+#include <thread>
+#include <sstream>
+
+#include <string.h>
+
+using std::chrono::duration;
+using std::chrono::time_point;
+using std::chrono::steady_clock;
+using std::chrono::milliseconds;
+using std::map;
+using std::ostream;
+using std::string;
+using std::stringstream;
+using std::this_thread::sleep_for;
+using std::vector;
+using nlohmann::json;
+
+static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, void *userdata)
+{
+	ostream *out = static_cast<ostream*>(userdata);
+	size_t bytes = 0;
+
+	if (nullptr == out)
+	{
+		return nmemb * size;
+	}
+
+	for (size_t idx = 0; idx < nmemb * size; ++idx)
+	{
+		(*out) << static_cast<char*>(ptr)[idx];
+		++bytes;
+	}
+
+	return bytes;
+}
+
+static size_t curl_header_func(void *ptr, size_t size, size_t nmemb, void *userdata)
+{
+	map<string, string> *out = static_cast<map<string, string> *>(userdata);
+	stringstream helper;
+	size_t bytes = 0;
+	string full;
+	string name;
+	string value;
+	size_t pos;
+	constexpr char whitespace[] = " \r\n\t\f\v";
+
+	if (nullptr == out)
+	{
+		return nmemb * size;
+	}
+
+	for (size_t idx = 0; idx < nmemb * size; ++idx)
+	{
+		helper << static_cast<char*>(ptr)[idx];
+		++bytes;
+	}
+
+	// Split into 2 strings
+	full = helper.str();
+	pos = full.find(':');
+	name = full.substr(0, pos);
+	value = "";
+
+	if (pos != string::npos)
+	{
+		value = full.substr(pos + 1);
+	}
+
+	// Clean whitespace
+	pos = name.find_first_not_of(whitespace);
+	name.erase(0, pos);
+	pos = name.find_last_not_of(whitespace);
+	name.erase(pos + 1);
+
+	pos = value.find_first_not_of(whitespace);
+	value.erase(0, pos);
+	pos = value.find_last_not_of(whitespace);
+	value.erase(pos + 1);
+
+	// Make header name lowercase
+	for (int i = 0; i < name.size(); ++i)
+	{
+		if ((name[i] >= 'A') && (name[i] <= 'Z'))
+		{
+			name[i] += 'a' - 'A';
+		}
+	}
+
+	// Insert header
+	(*out)[name] = value;
+
+	return bytes;
+}
+
+void CURL_DL::open_curl()
+{
+	m_handle = curl_easy_init();
+}
+
+void CURL_DL::check_rate_limit(const RateLimit& limit)
+{
+	if (!limit.m_name.empty() && limit.m_ms > 0)
+	{
+		if (m_limits.count(limit.m_name) != 0)
+		{
+			time_point<steady_clock> wait_end = m_limits[limit.m_name] + duration<int, std::milli>(limit.m_ms);
+			time_point<steady_clock> now = steady_clock::now();
+
+			if (now < wait_end)
+			{
+				sleep_for(wait_end - now);
+			}
+		}
+	}
+}
+
+void CURL_DL::save_rate_limit(const RateLimit& limit)
+{
+	if (!limit.m_name.empty())
+	{
+		m_limits[limit.m_name] = steady_clock::now();
+	}
+}
+
+CURL_DL::CURL_DL()
+: m_handle(nullptr)
+{
+}
+
+CURL_DL::~CURL_DL()
+{
+	if (m_handle != nullptr)
+	{
+		curl_easy_cleanup(m_handle);
+		m_handle = nullptr;
+	}
+}
+
+CURL_DL& CURL_DL::get_handle()
+{
+	static CURL_DL obj;
+	return obj;
+}
+
+bool CURL_DL::download(const string& url, ostream* out, RateLimit limit)
+{
+	return download(url, out, nullptr, nullptr, nullptr, limit);
+}
+
+bool CURL_DL::download(const string& url, ostream* out,
+	const vector<string> *headers, RateLimit limit)
+{
+	return download(url, out, headers, nullptr, nullptr, limit);
+}
+
+bool CURL_DL::download(const string& url, ostream* out,
+	const vector<string> *headers, map<string, string> *out_headers,
+	RateLimit limit)
+{
+	return download(url, out, headers, out_headers, nullptr, limit);
+}
+
+bool CURL_DL::download(const string& url, ostream* out,
+	const vector<string> *headers, map<string, string> *out_headers,
+	const map<string, string> *params, RateLimit limit)
+{
+	bool result = true;
+	CURLcode error;
+	struct curl_slist *header_list = nullptr;
+	long http_code;
+
+	if (nullptr == m_handle)
+	{
+		open_curl();
+	}
+
+	if (nullptr == m_handle)
+	{
+		return false;
+	}
+
+	curl_easy_reset(m_handle);
+
+	// Enable cookie engine
+	curl_easy_setopt(m_handle, CURLOPT_COOKIEFILE, "cookies.txt");
+
+	// Enable error messages
+	curl_easy_setopt(m_handle, CURLOPT_ERRORBUFFER, m_error);
+
+	// Set User-Agent
+	curl_easy_setopt(m_handle, CURLOPT_USERAGENT, "Internedko Downloader");
+
+	if ((nullptr == params) || (params->count("no-redir") == 0))
+	{
+		// Set Auto Follow (Max 32 Times)
+		curl_easy_setopt(m_handle, CURLOPT_FOLLOWLOCATION, 1);
+		curl_easy_setopt(m_handle, CURLOPT_MAXREDIRS, 32);
+	}
+	else
+	{
+		curl_easy_setopt(m_handle, CURLOPT_FOLLOWLOCATION, 0);
+	}
+
+	// Allow only HTTP and HTTPS (STR after 7.85.0)
+	// curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS_STR, "http,https");
+	curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+
+	// Do Not Return "OK" On 4xx And 5xx
+	curl_easy_setopt(m_handle, CURLOPT_FAILONERROR, 0);
+
+	// Set Output Func
+	curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, curl_write_func);
+	curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, out);
+
+	// Set URL
+	curl_easy_setopt(m_handle, CURLOPT_URL, url.c_str());
+
+	// Set Get
+	curl_easy_setopt(m_handle, CURLOPT_HTTPGET, 1);
+
+	// Set Headers
+	if (nullptr != headers)
+	{
+		for (const string& s : *headers)
+		{
+			header_list = curl_slist_append(header_list, s.c_str());
+		}
+	}
+	curl_easy_setopt(m_handle, CURLOPT_HTTPHEADER, header_list);
+
+	if (nullptr != out_headers)
+	{
+		curl_easy_setopt(m_handle, CURLOPT_HEADERFUNCTION, curl_header_func);
+		curl_easy_setopt(m_handle, CURLOPT_HEADERDATA, out_headers);
+	}
+
+	// Set Parameters
+	if (nullptr != params)
+	{
+		// Username
+		if (params->count("user") != 0)
+		{
+			curl_easy_setopt(m_handle, CURLOPT_USERNAME, (*params).at("user").c_str());
+		}
+
+		// Password
+		if (params->count("pass") != 0)
+		{
+			curl_easy_setopt(m_handle, CURLOPT_PASSWORD, (*params).at("pass").c_str());
+		}
+	}
+
+	check_rate_limit(limit);
+
+	error = curl_easy_perform(m_handle);
+	if (error != CURLE_OK)
+	{
+		result = false;
+	}
+
+	error = curl_easy_getinfo(m_handle, CURLINFO_RESPONSE_CODE, &http_code);
+	if (error != CURLE_OK)
+	{
+		result = false;
+	}
+
+	if (http_code >= 400)
+	{
+		result = false;
+	}
+
+	save_rate_limit(limit);
+
+	// Cleanup
+	if (nullptr != header_list)
+	{
+		curl_slist_free_all(header_list);
+	}
+	header_list = nullptr;
+
+	return result;
+}
+
+bool CURL_DL::post_json(const string& url, json& j, ostream* out, RateLimit limit)
+{
+	return post_json(url, j, out, nullptr, limit);
+}
+
+bool CURL_DL::post_json(const string& url, json& j, ostream* out,
+	const vector<string> *headers, RateLimit limit)
+{
+	bool result = true;
+	CURLcode error;
+	struct curl_slist *header_list = nullptr;
+
+	if (nullptr == m_handle)
+	{
+		open_curl();
+	}
+
+	if (nullptr == m_handle)
+	{
+		return false;
+	}
+
+	curl_easy_reset(m_handle);
+
+	curl_easy_setopt(m_handle, CURLOPT_COOKIEFILE, "");
+
+	curl_easy_setopt(m_handle, CURLOPT_ERRORBUFFER, m_error);
+
+	// Set User-Agent
+	curl_easy_setopt(m_handle, CURLOPT_USERAGENT, "Internedko Archiver");
+
+	// Set Auto Follow (Max 32 Times)
+	curl_easy_setopt(m_handle, CURLOPT_FOLLOWLOCATION, 1);
+	curl_easy_setopt(m_handle, CURLOPT_MAXREDIRS, 32);
+
+	// Allow only HTTP and HTTPS (STR after 7.85.0)
+	// curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS_STR, "http,https");
+	curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+
+	// Do Not Return "OK" On 4xx And 5xx
+	curl_easy_setopt(m_handle, CURLOPT_FAILONERROR, 1);
+
+	// Set Output Func
+	curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, curl_write_func);
+	curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, out);
+
+	// Set URL
+	curl_easy_setopt(m_handle, CURLOPT_URL, url.c_str());
+
+	// Set Post
+	curl_easy_setopt(m_handle, CURLOPT_POST, 1);
+
+	// Set Headers
+	header_list = curl_slist_append(header_list, "Content-Type: application/json");
+	if (nullptr != headers)
+	{
+		for (const string& s : *headers)
+		{
+			header_list = curl_slist_append(header_list, s.c_str());
+		}
+	}
+	curl_easy_setopt(m_handle, CURLOPT_HTTPHEADER, header_list);
+
+	// Set Data
+	curl_easy_setopt(m_handle, CURLOPT_COPYPOSTFIELDS, j.dump().c_str());
+
+	check_rate_limit(limit);
+
+	error = curl_easy_perform(m_handle);
+	if (error != CURLE_OK)
+	{
+		result = false;
+	}
+
+	save_rate_limit(limit);
+
+	// Cleanup
+	curl_slist_free_all(header_list);
+	header_list = nullptr;
+
+	return result;
+}
+
+string CURL_DL::get_error() const
+{
+	return m_error;
+}
+
+string CURL_DL::url_encode(const string& input)
+{
+	string result;
+	char hex[3];
+
+	hex[2] = '\0';
+
+	for (int i = 0; i < input.size(); ++i)
+	{
+		if (((input[i] >= ' ') && (input[i] < '0')) ||
+			((input[i] > '9') && (input[i] < 'A')) ||
+			((input[i] > 'Z') && (input[i] < 'a')) ||
+			((input[i] > 'z') && (input[i] <= '~')))
+		{
+			result += '%';
+			sprintf(hex, "%X", input[i]);
+			result += hex;
+		}
+		else
+		{
+			result += input[i];
+		}
+	}
+
+	return result;
+}
+
+string CURL_DL::calc_redir(string url, const string& location)
+{
+	size_t pos;
+
+	pos = url.find('?');
+	if (string::npos != pos)
+	{
+		url.erase(pos);
+	}
+
+	if (location[0] == '/')
+	{
+		pos = url.find("://");
+		if (string::npos == pos)
+		{
+			pos = url.find("/");
+		}
+		else
+		{
+			pos = url.find("/", pos + 3);
+		}
+
+		// Pos now points to first / after ://
+		if (string::npos != pos)
+		{
+			url.erase(pos);
+		}
+
+		url += location;
+		return url;
+	}
+	else
+	{
+		return location;
+	}
+}

curl_dl.h

@@ -0,0 +1,74 @@
+#ifndef CURL_DL_H_
+#define CURL_DL_H_
+
+#include <chrono>
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <curl/curl.h>
+
+#include "json.hpp"
+
+class CURL_DL
+{
+public:
+	class RateLimit
+	{
+	friend class CURL_DL;
+	private:
+		std::string m_name;
+		uint32_t m_ms;
+
+	public:
+		RateLimit(std::string name = "", uint32_t ms = 0)
+		: m_name(name), m_ms(ms)
+		{}
+	};
+
+private:
+	CURL *m_handle;
+
+	std::map<std::string, std::chrono::time_point<std::chrono::steady_clock>> m_limits;
+
+	char m_error[CURL_ERROR_SIZE];
+
+	void open_curl();
+
+	void check_rate_limit(const RateLimit& limit);
+	void save_rate_limit(const RateLimit& limit);
+
+	CURL_DL();
+	CURL_DL(const CURL_DL&) = delete;
+	CURL_DL(CURL_DL&&) = delete;
+
+public:
+	~CURL_DL();
+	static CURL_DL& get_handle();
+
+	bool download(const std::string& url, std::ostream* out,
+		RateLimit limit = RateLimit());
+	bool download(const std::string& url, std::ostream* out,
+		const std::vector<std::string> *headers, RateLimit limit = RateLimit());
+	bool download(const std::string& url, std::ostream* out,
+		const std::vector<std::string> *headers,
+		std::map<std::string, std::string> *out_headers,
+		RateLimit limit = RateLimit());
+	bool download(const std::string& url, std::ostream* out,
+		const std::vector<std::string> *headers,
+		std::map<std::string, std::string> *out_headers,
+		const std::map<std::string, std::string> *params,
+		RateLimit limit = RateLimit());
+	bool post_json(const std::string& url, nlohmann::json& j, std::ostream* out,
+		RateLimit limit = RateLimit());
+	bool post_json(const std::string& url, nlohmann::json& j, std::ostream* out,
+		const std::vector<std::string> *headers, RateLimit limit = RateLimit());
+
+	std::string get_error() const;
+
+	static std::string url_encode(const std::string& input);
+	static std::string calc_redir(std::string url, const std::string& location);
+};
+
+#endif // CURL_DL_H_

example_download.json

@@ -1,7 +0,0 @@
-{
-	"comment": "This file will attempt to download MCLP001, MCLP001-X, MCLP002, ... MCLP010-X",
-	"prefix":		"MCLP",
-	"suffix_try":	"-X",
-	"start":		1,
-	"end":			10
-}

example_login.json

@@ -1,4 +0,0 @@
-{
-	"Email": "USER",
-	"Password": "PASS"
-}

monstercat_dl.cpp

@@ -0,0 +1,599 @@
+#include "monstercat_dl.h"
+#include "common.h"
+#include "curl_dl.h"
+
+#include <algorithm>
+#include <fstream>
+#include <sstream>
+
+using std::endl;
+using std::map;
+using std::ofstream;
+using std::sort;
+using std::string;
+using std::stringstream;
+using std::to_string;
+
+using nlohmann::json;
+
+string Monstercat_DL::calc_proper_artist(const string& artist_raw)
+{
+	size_t pos;
+	string result;
+
+	pos = artist_raw.find("feat.");
+	if (string::npos == pos)
+	{
+		return artist_raw;
+	}
+
+	result = artist_raw.substr(0, pos);
+	result = trim_whitespace(result);
+
+	return result;
+}
+
+string Monstercat_DL::calc_proper_title(const string& artist_raw,
+	const string& title_raw, const string& version_raw)
+{
+	size_t pos;
+	string result;
+
+	result = trim_whitespace(title_raw);
+
+	pos = artist_raw.find("feat.");
+	if (string::npos != pos)
+	{
+		result += " (";
+		result += trim_whitespace(artist_raw.substr(pos));
+		result += ")";
+	}
+
+	if (!version_raw.empty())
+	{
+		result += " (";
+		result += trim_whitespace(version_raw);
+		result += ")";
+	}
+
+	return result;
+}
+
+Monstercat_DL::Monstercat_DL()
+: m_log(&std::cout), m_base_url("https://player.monstercat.app/api/"),
+	m_is_logged_in(false)
+{}
+
+Monstercat_DL::~Monstercat_DL()
+{
+}
+
+bool Monstercat_DL::login(const string& user, const string& pass)
+{
+	CURL_DL& curl = CURL_DL::get_handle();
+	bool ok;
+	json data;
+	string url;
+	stringstream out;
+
+	data["Email"] = user;
+	data["Password"] = pass;
+
+	url = m_base_url;
+	url += "sign-in";
+
+	ok = curl.post_json(url, data, &out);
+	if (!ok)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not post json" << endl;
+			*m_log << "CURL:" << curl.get_error() << endl;
+		}
+		return false;
+	}
+
+	data.clear();
+	out >> data;
+
+	if (data.contains("Needs2FA") &&
+		data["Needs2FA"].is_boolean() &&
+		data["Needs2FA"] == false)
+	{
+		m_is_logged_in = true;
+		return true;
+	}
+
+	return false;
+}
+
+bool Monstercat_DL::logout()
+{
+	CURL_DL& curl = CURL_DL::get_handle();
+	bool ok;
+	json data;
+	string url;
+
+	if (!m_is_logged_in)
+	{
+		return false;
+	}
+
+	url = m_base_url;
+	url += "sign-out";
+
+	ok = curl.post_json(url, data, nullptr);
+	if (!ok)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not post json" << endl;
+			*m_log << "CURL:" << curl.get_error() << endl;
+		}
+		return false;
+	}
+
+	return ok;
+}
+
+json Monstercat_DL::get_release_json(const string& catalog_id)
+{
+	CURL_DL& curl = CURL_DL::get_handle();
+	bool ok;
+	json result;
+	string url;
+	stringstream out;
+
+	url = m_base_url;
+	url += "catalog/release/";
+	url += catalog_id;
+
+	ok = curl.download(url, &out);
+	if (!ok)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not download json" << endl;
+			*m_log << "CURL:" << curl.get_error() << endl;
+		}
+		return result;
+	}
+
+	out >> result;
+	return result;
+}
+
+json Monstercat_DL::get_browse_json(const string& release_id)
+{
+	CURL_DL& curl = CURL_DL::get_handle();
+	bool ok;
+	json result;
+	string url;
+	stringstream out;
+
+	url = m_base_url;
+	url += "catalog/browse?offset=0&limit=0&search=&sort=&nogold=false&releaseId=";
+	url += release_id;
+
+	ok = curl.download(url, &out);
+	if (!ok)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not download json" << endl;
+			*m_log << "CURL:" << curl.get_error() << endl;
+		}
+		return result;
+	}
+
+	out >> result;
+	return result;
+}
+
+Release Monstercat_DL::parse_release_json(const json& release_json)
+{
+	Release result;
+	json release_object;
+	json tracks_object;
+	string artist_raw;
+	string title_raw;
+	string version_raw;
+	Track temp_track;
+
+	if (release_json.contains("Release"))
+	{
+		release_object = release_json["Release"];
+	}
+	else
+	{
+		return result;
+	}
+
+	// Parse Release
+	json_extract(release_object, "CatalogId", result.catalog_id);
+	json_extract(release_object, "Id", result.id);
+	json_extract(release_object, "Type", result.type);
+	json_extract(release_object, "ReleaseDate", result.release_date);
+	{
+		size_t pos = result.release_date.find('T');
+		if (pos != string::npos)
+		{
+			result.release_date.erase(pos);
+		}
+	}
+
+	json_extract(release_object, "ArtistsTitle", artist_raw);
+	json_extract(release_object, "Title", title_raw);
+	json_extract(release_object, "Version", version_raw);
+
+	result.artist = calc_proper_artist(artist_raw);
+	result.title = calc_proper_title(artist_raw, title_raw, version_raw);
+
+	if (release_json.contains("Tracks") && release_json["Tracks"].is_array())
+	{
+		tracks_object = release_json["Tracks"];
+	}
+	else
+	{
+		return result;
+	}
+
+	// Parse Tracks
+	for (json& curr_track : tracks_object)
+	{
+		if (curr_track.contains("TrackNumber") &&
+			curr_track["TrackNumber"].is_number_integer())
+		{
+			temp_track.number = curr_track["TrackNumber"];
+		}
+		json_extract(curr_track, "Id", temp_track.id);
+
+		json_extract(curr_track, "ArtistsTitle", artist_raw);
+		json_extract(curr_track, "Title", title_raw);
+		json_extract(curr_track, "Version", version_raw);
+
+		temp_track.artist = calc_proper_artist(artist_raw);
+		temp_track.title = calc_proper_title(artist_raw, title_raw, version_raw);
+
+		result.tracks.push_back(temp_track);
+	}
+
+	// Sort tracks by number
+	sort(result.tracks.begin(), result.tracks.end(), [](const Track& t1, const Track& t2)
+	{
+		return t1.number < t2.number;
+	});
+
+	return result;
+}
+
+void Monstercat_DL::add_extended_mixes(Release& release, const json& browse_json)
+{
+	int track_num;
+	json data_obj;
+	json file_obj;
+	Track *track;
+	string mime_type;
+
+	if (browse_json.contains("Data") && browse_json["Data"].is_array())
+	{
+		data_obj = browse_json["Data"];
+	}
+
+	for (json& track_json : data_obj)
+	{
+		// Get the track number
+		track_num = 0;
+		if (track_json.contains("TrackNumber") && track_json["TrackNumber"].is_number_integer())
+		{
+			track_num = track_json["TrackNumber"];
+		}
+
+		// File must exist and be an object
+		if (!track_json.contains("File") || !track_json["File"].is_object())
+		{
+			continue;
+		}
+
+		// Find track
+		track = nullptr;
+		for (int i = 0; i < release.tracks.size(); ++i)
+		{
+			if (release.tracks[i].number == track_num)
+			{
+				track = &(release.tracks[i]);
+				break;
+			}
+		}
+
+		if (nullptr == track)
+		{
+			if (nullptr != m_log)
+			{
+				*m_log << __PRETTY_FUNCTION__ << endl;
+				*m_log << "Could not find track number " << track_num << " for catalog id " << release.catalog_id << endl;
+			}
+			return;
+		}
+
+		file_obj = track_json["File"];
+		json_extract(file_obj, "Id", track->extended_mix_file_id);
+		json_extract(file_obj, "MimeType", mime_type);
+
+		if ("audio/wav" == mime_type)
+		{
+			track->extended_mix_extension = ".wav";
+		}
+		else
+		{
+			if (nullptr != m_log)
+			{
+				*m_log << __PRETTY_FUNCTION__ << endl;
+				*m_log << "Unknown MIME type for catalog id " << release.catalog_id << " - " << mime_type << endl;
+			}
+		}
+		return;
+	}
+}
+
+string Monstercat_DL::calc_release_folder(const Release& release, const string& main_folder)
+{
+	string result;
+	string folder_name;
+
+	ensure_folder(main_folder, release.type);
+
+	folder_name = release.release_date;
+	folder_name += " - ";
+	folder_name += clean_filename(release.catalog_id);
+	folder_name += " - ";
+	folder_name += clean_filename(release.artist);
+	folder_name += " - ";
+	folder_name += clean_filename(release.title);
+
+	result = build_fname(main_folder, release.type, folder_name);
+	ensure_folder(result, "");
+
+	return result;
+}
+
+string Monstercat_DL::calc_track_filename(const Release& release, int track_num)
+{
+	string result;
+	bool use_number;
+	bool use_artist;
+	const Track *track = nullptr;
+
+	// Find the track
+	for (int i = 0; i < release.tracks.size(); ++i)
+	{
+		if (release.tracks[i].number == track_num)
+		{
+			track = &(release.tracks[i]);
+			break;
+		}
+	}
+
+	if (nullptr == track)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Cannot find track number " << track_num << " in catalog id " << release.catalog_id << endl;
+		}
+
+		return "";
+	}
+
+	// Determine whether number is needed
+	if (1 == release.tracks.size())
+	{
+		use_number = false;
+	}
+	else
+	{
+		use_number = true;
+	}
+
+	// Determine whether artist is needed
+	if (track->artist == release.artist)
+	{
+		use_artist = false;
+	}
+	else
+	{
+		use_artist = true;
+	}
+
+	// Build filename
+	result = "";
+	if (use_number)
+	{
+		// Always use 2 digits
+		result += i_to_str(track->number, 2, '0');
+		result += " - ";
+	}
+
+	if (use_artist)
+	{
+		result += track->artist;
+		result += " - ";
+	}
+
+	result += track->title;
+
+	return clean_filename(result);
+}
+
+bool Monstercat_DL::download_cover(const string& catalog_id, const string& path)
+{
+	CURL_DL& curl = CURL_DL::get_handle();
+	bool ok;
+	string url;
+	stringstream out;
+	ofstream out_file;
+	map<string, string> out_headers;
+	string filepath;
+
+	url = "https://www.monstercat.com/release/";
+	url += catalog_id;
+	url += "/cover";
+
+	ok = curl.download(url, &out, nullptr, &out_headers);
+	if (!ok)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not download image" << endl;
+			*m_log << "CURL:" << curl.get_error() << endl;
+		}
+		return false;
+	}
+
+	filepath = build_fname(path, "", "Cover");
+	if (0 == out_headers.count("content-type"))
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "No content-type" << endl;
+		}
+	}
+	else
+	{
+		if (out_headers["content-type"] == "image/jpeg")
+		{
+			filepath += ".jpg";
+		}
+		else if (out_headers["content-type"] == "image/png")
+		{
+			filepath += ".png";
+		}
+		else
+		{
+			if (nullptr != m_log)
+			{
+				*m_log << __PRETTY_FUNCTION__ << endl;
+				*m_log << "Unknown Content-Type for cover - " << out_headers["content-type"] << endl;
+			}
+		}
+	}
+
+	out_file.open(filepath, std::ios::binary);
+	if (!out_file.is_open())
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not open file for write" << endl;
+			*m_log << filepath << endl;
+		}
+		return false;
+	}
+
+	out_file << out.rdbuf();
+	out_file.close();
+
+	return true;
+}
+
+bool Monstercat_DL::download_track(const string& release_id,
+	const string& track_id, const string& filepath, bool is_mp3)
+{
+	CURL_DL& curl = CURL_DL::get_handle();
+	bool ok;
+	string url;
+	stringstream out_data;
+	ofstream out_file;
+
+	url = m_base_url;
+	url += "release/";
+	url += release_id;
+	url += "/track-download/";
+	url += track_id;
+	if (is_mp3)
+	{
+		url += "?format=mp3_320";
+	}
+	else
+	{
+		url += "?format=flac";
+	}
+
+	ok = curl.download(url, &out_data);
+	if (!ok)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not download track" << endl;
+			*m_log << "CURL:" << curl.get_error() << endl;
+		}
+		return false;
+	}
+
+	out_file.open(filepath, std::ios::binary);
+	if (!out_file.is_open())
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not open file for write" << endl;
+			*m_log << filepath << endl;
+		}
+		return false;
+	}
+
+	out_file << out_data.rdbuf();
+	out_file.close();
+
+	return true;
+}
+
+bool Monstercat_DL::download_file(const string& file_id, const string& filepath)
+{
+	CURL_DL& curl = CURL_DL::get_handle();
+	bool ok;
+	string url;
+	stringstream out_data;
+	ofstream out_file;
+
+	url = m_base_url;
+	url += "file/";
+	url += file_id;
+	url += "/open?download=true";
+
+	ok = curl.download(url, &out_data);
+	if (!ok)
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not download track" << endl;
+			*m_log << "CURL:" << curl.get_error() << endl;
+		}
+		return false;
+	}
+
+	out_file.open(filepath, std::ios::binary);
+	if (!out_file.is_open())
+	{
+		if (nullptr != m_log)
+		{
+			*m_log << __PRETTY_FUNCTION__ << endl;
+			*m_log << "Could not open file for write" << endl;
+			*m_log << filepath << endl;
+		}
+		return false;
+	}
+
+	out_file << out_data.rdbuf();
+	out_file.close();
+
+	return true;
+}

monstercat_dl.h

@@ -0,0 +1,72 @@
+#ifndef MONSTERCAT_DL_H_
+#define MONSTERCAT_DL_H_
+
+#include <iostream>
+#include <string>
+#include <vector>
+#include "json.hpp"
+
+constexpr bool FORMAT_MP3 = true;
+constexpr bool FORMAT_FLAC = false;
+
+struct Track
+{
+public:
+	int number;
+	std::string artist;
+	std::string title;
+
+	std::string id;
+	std::string extended_mix_file_id;
+	std::string extended_mix_extension;
+};
+
+struct Release
+{
+	std::string type;
+	std::string release_date;
+	std::string artist;
+	std::string title;
+	std::vector<Track> tracks;
+
+	std::string id;
+	std::string catalog_id;
+};
+
+class Monstercat_DL
+{
+private:
+	std::ostream *m_log;
+
+	std::string m_base_url;
+
+	bool m_is_logged_in;
+
+	std::string calc_proper_artist(const std::string& artist_raw);
+	std::string calc_proper_title(const std::string& artist_raw,
+		const std::string& title_raw, const std::string& version_raw);
+
+public:
+	Monstercat_DL();
+	~Monstercat_DL();
+
+	bool login(const std::string& user, const std::string& pass);
+	bool logout();
+
+	nlohmann::json get_release_json(const std::string& catalog_id);
+	nlohmann::json get_browse_json(const std::string& release_id);
+
+	Release parse_release_json(const nlohmann::json& release_json);
+	void add_extended_mixes(Release& release, const nlohmann::json& browse_json);
+
+	std::string calc_release_folder(const Release& release, const std::string& main_folder);
+	std::string calc_track_filename(const Release& release, int track_num);
+
+	bool download_cover(const std::string& catalog_id, const std::string& path);
+	bool download_track(const std::string& release_id,
+		const std::string& track_id, const std::string& filepath, bool is_mp3);
+	bool download_file(const std::string& file_id, const std::string& filepath);
+
+};
+
+#endif // MONSTERCAT_DL_H_