diff --git a/curl_dl.cpp b/curl_dl.cpp new file mode 100644 index 0000000..78e92ba --- /dev/null +++ b/curl_dl.cpp @@ -0,0 +1,437 @@ +#include "curl_dl.h" + +#include +#include + +#include + +using std::chrono::duration; +using std::chrono::time_point; +using std::chrono::steady_clock; +using std::chrono::milliseconds; +using std::map; +using std::ostream; +using std::string; +using std::stringstream; +using std::this_thread::sleep_for; +using std::vector; +using nlohmann::json; + +static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + ostream *out = static_cast(userdata); + size_t bytes = 0; + + if (nullptr == out) + { + return nmemb * size; + } + + for (size_t idx = 0; idx < nmemb * size; ++idx) + { + (*out) << static_cast(ptr)[idx]; + ++bytes; + } + + return bytes; +} + +static size_t curl_header_func(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + map *out = static_cast *>(userdata); + stringstream helper; + size_t bytes = 0; + string full; + string name; + string value; + size_t pos; + constexpr char whitespace[] = " \r\n\t\f\v"; + + if (nullptr == out) + { + return nmemb * size; + } + + for (size_t idx = 0; idx < nmemb * size; ++idx) + { + helper << static_cast(ptr)[idx]; + ++bytes; + } + + // Split into 2 strings + full = helper.str(); + pos = full.find(':'); + name = full.substr(0, pos); + value = ""; + + if (pos != string::npos) + { + value = full.substr(pos + 1); + } + + // Clean whitespace + pos = name.find_first_not_of(whitespace); + name.erase(0, pos); + pos = name.find_last_not_of(whitespace); + name.erase(pos + 1); + + pos = value.find_first_not_of(whitespace); + value.erase(0, pos); + pos = value.find_last_not_of(whitespace); + value.erase(pos + 1); + + // Make header name lowercase + for (int i = 0; i < name.size(); ++i) + { + if ((name[i] >= 'A') && (name[i] <= 'Z')) + { + name[i] += 'a' - 'A'; + } + } + + // Insert header + (*out)[name] = value; + + return bytes; +} + +void CURL_DL::open_curl() +{ + m_handle = curl_easy_init(); +} + +void CURL_DL::check_rate_limit(const RateLimit& limit) +{ + if (!limit.m_name.empty() && limit.m_ms > 0) + { + if (m_limits.count(limit.m_name) != 0) + { + time_point wait_end = m_limits[limit.m_name] + duration(limit.m_ms); + time_point now = steady_clock::now(); + + if (now < wait_end) + { + sleep_for(wait_end - now); + } + } + } +} + +void CURL_DL::save_rate_limit(const RateLimit& limit) +{ + if (!limit.m_name.empty()) + { + m_limits[limit.m_name] = steady_clock::now(); + } +} + +CURL_DL::CURL_DL() +: m_handle(nullptr) +{ +} + +CURL_DL::~CURL_DL() +{ + if (m_handle != nullptr) + { + curl_easy_cleanup(m_handle); + m_handle = nullptr; + } +} + +CURL_DL& CURL_DL::get_handle() +{ + static CURL_DL obj; + return obj; +} + +bool CURL_DL::download(const string& url, ostream* out, RateLimit limit) +{ + return download(url, out, nullptr, nullptr, nullptr, limit); +} + +bool CURL_DL::download(const string& url, ostream* out, + const vector *headers, RateLimit limit) +{ + return download(url, out, headers, nullptr, nullptr, limit); +} + +bool CURL_DL::download(const string& url, ostream* out, + const vector *headers, map *out_headers, + RateLimit limit) +{ + return download(url, out, headers, out_headers, nullptr, limit); +} + +bool CURL_DL::download(const string& url, ostream* out, + const vector *headers, map *out_headers, + const map *params, RateLimit limit) +{ + bool result = true; + CURLcode error; + struct curl_slist *header_list = nullptr; + long http_code; + + if (nullptr == m_handle) + { + open_curl(); + } + + if (nullptr == m_handle) + { + return false; + } + + curl_easy_reset(m_handle); + + // Enable cookie engine + curl_easy_setopt(m_handle, CURLOPT_COOKIEFILE, ""); + + // Enable error messages + curl_easy_setopt(m_handle, CURLOPT_ERRORBUFFER, m_error); + + // Set User-Agent + curl_easy_setopt(m_handle, CURLOPT_USERAGENT, "Internedko Archiver"); + + if ((nullptr == params) || (params->count("no-redir") == 0)) + { + // Set Auto Follow (Max 32 Times) + curl_easy_setopt(m_handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(m_handle, CURLOPT_MAXREDIRS, 32); + } + else + { + curl_easy_setopt(m_handle, CURLOPT_FOLLOWLOCATION, 0); + } + + // Allow only HTTP and HTTPS (STR after 7.85.0) + // curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS_STR, "http,https"); + curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + + // Do Not Return "OK" On 4xx And 5xx + curl_easy_setopt(m_handle, CURLOPT_FAILONERROR, 0); + + // Set Output Func + curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, curl_write_func); + curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, out); + + // Set URL + curl_easy_setopt(m_handle, CURLOPT_URL, url.c_str()); + + // Set Get + curl_easy_setopt(m_handle, CURLOPT_HTTPGET, 1); + + // Set Headers + if (nullptr != headers) + { + for (const string& s : *headers) + { + header_list = curl_slist_append(header_list, s.c_str()); + } + } + curl_easy_setopt(m_handle, CURLOPT_HTTPHEADER, header_list); + + if (nullptr != out_headers) + { + curl_easy_setopt(m_handle, CURLOPT_HEADERFUNCTION, curl_header_func); + curl_easy_setopt(m_handle, CURLOPT_HEADERDATA, out_headers); + } + + // Set Parameters + if (nullptr != params) + { + // Username + if (params->count("user") != 0) + { + curl_easy_setopt(m_handle, CURLOPT_USERNAME, (*params).at("user").c_str()); + } + + // Password + if (params->count("pass") != 0) + { + curl_easy_setopt(m_handle, CURLOPT_PASSWORD, (*params).at("pass").c_str()); + } + } + + check_rate_limit(limit); + + error = curl_easy_perform(m_handle); + if (error != CURLE_OK) + { + result = false; + } + + error = curl_easy_getinfo(m_handle, CURLINFO_RESPONSE_CODE, &http_code); + if (error != CURLE_OK) + { + result = false; + } + + if (http_code >= 400) + { + result = false; + } + + save_rate_limit(limit); + + // Cleanup + if (nullptr != header_list) + { + curl_slist_free_all(header_list); + } + header_list = nullptr; + + return result; +} + +bool CURL_DL::post_json(const string& url, json& j, ostream* out, RateLimit limit) +{ + return post_json(url, j, out, nullptr, limit); +} + +bool CURL_DL::post_json(const string& url, json& j, ostream* out, + const vector *headers, RateLimit limit) +{ + bool result = true; + CURLcode error; + struct curl_slist *header_list = nullptr; + + if (nullptr == m_handle) + { + open_curl(); + } + + if (nullptr == m_handle) + { + return false; + } + + curl_easy_reset(m_handle); + + curl_easy_setopt(m_handle, CURLOPT_COOKIEFILE, ""); + + curl_easy_setopt(m_handle, CURLOPT_ERRORBUFFER, m_error); + + // Set User-Agent + curl_easy_setopt(m_handle, CURLOPT_USERAGENT, "Internedko Archiver"); + + // Set Auto Follow (Max 32 Times) + curl_easy_setopt(m_handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(m_handle, CURLOPT_MAXREDIRS, 32); + + // Allow only HTTP and HTTPS (STR after 7.85.0) + // curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS_STR, "http,https"); + curl_easy_setopt(m_handle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + + // Do Not Return "OK" On 4xx And 5xx + curl_easy_setopt(m_handle, CURLOPT_FAILONERROR, 1); + + // Set Output Func + curl_easy_setopt(m_handle, CURLOPT_WRITEFUNCTION, curl_write_func); + curl_easy_setopt(m_handle, CURLOPT_WRITEDATA, out); + + // Set URL + curl_easy_setopt(m_handle, CURLOPT_URL, url.c_str()); + + // Set Post + curl_easy_setopt(m_handle, CURLOPT_POST, 1); + + // Set Headers + header_list = curl_slist_append(header_list, "Content-Type: application/json"); + if (nullptr != headers) + { + for (const string& s : *headers) + { + header_list = curl_slist_append(header_list, s.c_str()); + } + } + curl_easy_setopt(m_handle, CURLOPT_HTTPHEADER, header_list); + + // Set Data + curl_easy_setopt(m_handle, CURLOPT_COPYPOSTFIELDS, j.dump().c_str()); + + check_rate_limit(limit); + + error = curl_easy_perform(m_handle); + if (error != CURLE_OK) + { + result = false; + } + + save_rate_limit(limit); + + // Cleanup + curl_slist_free_all(header_list); + header_list = nullptr; + + return result; +} + +string CURL_DL::get_error() const +{ + return m_error; +} + +string CURL_DL::url_encode(const string& input) +{ + string result; + char hex[3]; + + hex[2] = '\0'; + + for (int i = 0; i < input.size(); ++i) + { + if (((input[i] >= ' ') && (input[i] < '0')) || + ((input[i] > '9') && (input[i] < 'A')) || + ((input[i] > 'Z') && (input[i] < 'a')) || + ((input[i] > 'z') && (input[i] <= '~'))) + { + result += '%'; + sprintf(hex, "%X", input[i]); + result += hex; + } + else + { + result += input[i]; + } + } + + return result; +} + +string CURL_DL::calc_redir(string url, const string& location) +{ + size_t pos; + + pos = url.find('?'); + if (string::npos != pos) + { + url.erase(pos); + } + + if (location[0] == '/') + { + pos = url.find("://"); + if (string::npos == pos) + { + pos = url.find("/"); + } + else + { + pos = url.find("/", pos + 3); + } + + // Pos now points to first / after :// + if (string::npos != pos) + { + url.erase(pos); + } + + url += location; + return url; + } + else + { + return location; + } +} diff --git a/curl_dl.h b/curl_dl.h new file mode 100644 index 0000000..190179d --- /dev/null +++ b/curl_dl.h @@ -0,0 +1,74 @@ +#ifndef CURL_DL_H_ +#define CURL_DL_H_ + +#include +#include +#include +#include +#include + +#include + +#include "json.hpp" + +class CURL_DL +{ +public: + class RateLimit + { + friend class CURL_DL; + private: + std::string m_name; + uint32_t m_ms; + + public: + RateLimit(std::string name = "", uint32_t ms = 0) + : m_name(name), m_ms(ms) + {} + }; + +private: + CURL *m_handle; + + std::map> m_limits; + + char m_error[CURL_ERROR_SIZE]; + + void open_curl(); + + void check_rate_limit(const RateLimit& limit); + void save_rate_limit(const RateLimit& limit); + + CURL_DL(); + CURL_DL(const CURL_DL&) = delete; + CURL_DL(CURL_DL&&) = delete; + +public: + ~CURL_DL(); + static CURL_DL& get_handle(); + + bool download(const std::string& url, std::ostream* out, + RateLimit limit = RateLimit()); + bool download(const std::string& url, std::ostream* out, + const std::vector *headers, RateLimit limit = RateLimit()); + bool download(const std::string& url, std::ostream* out, + const std::vector *headers, + std::map *out_headers, + RateLimit limit = RateLimit()); + bool download(const std::string& url, std::ostream* out, + const std::vector *headers, + std::map *out_headers, + const std::map *params, + RateLimit limit = RateLimit()); + bool post_json(const std::string& url, nlohmann::json& j, std::ostream* out, + RateLimit limit = RateLimit()); + bool post_json(const std::string& url, nlohmann::json& j, std::ostream* out, + const std::vector *headers, RateLimit limit = RateLimit()); + + std::string get_error() const; + + static std::string url_encode(const std::string& input); + static std::string calc_redir(std::string url, const std::string& location); +}; + +#endif // CURL_DL_H_ diff --git a/monstercat_dl.cpp b/monstercat_dl.cpp new file mode 100644 index 0000000..3f03f24 --- /dev/null +++ b/monstercat_dl.cpp @@ -0,0 +1,178 @@ +#include "monstercat_dl.h" +#include "curl_dl.h" + +#include +#include + +using std::endl; +using std::map; +using std::ofstream; +using std::string; +using std::stringstream; + +using nlohmann::json; + +Monstercat_DL::Monstercat_DL() +: log(&std::cout), m_base_url("https://player.monstercat.app/api/"), + m_is_logged_in(false) +{} + +Monstercat_DL::~Monstercat_DL() +{ +} + +bool Monstercat_DL::login(const string& user, const string& pass) +{ + CURL_DL& curl = CURL_DL::get_handle(); + bool ok; + json data; + string url; + stringstream out; + + data["Email"] = user; + data["Password"] = pass; + + url = m_base_url; + url += "sign-in"; + + ok = curl.post_json(url, data, &out); + if (!ok) + { + if (nullptr != log) + { + *log << "Could not post json" << endl; + *log << "CURL:" << curl.get_error() << endl; + } + return false; + } + + data.clear(); + out >> data; + + if (data.contains("Needs2FA") && + data["Needs2FA"].is_boolean() && + data["Needs2FA"] == false) + { + m_is_logged_in = true; + return true; + } + + return false; +} + +bool Monstercat_DL::logout() +{ + CURL_DL& curl = CURL_DL::get_handle(); + bool ok; + json data; + string url; + + if (!m_is_logged_in) + { + return false; + } + + url = m_base_url; + url += "sign-out"; + + ok = curl.post_json(url, data, nullptr); + if (!ok) + { + if (nullptr != log) + { + *log << "Could not post json" << endl; + *log << "CURL:" << curl.get_error() << endl; + } + return false; + } + + return ok; +} + +json Monstercat_DL::get_release_json(const string& catalog_id) +{ + CURL_DL& curl = CURL_DL::get_handle(); + bool ok; + json result; + string url; + stringstream out; + + url = m_base_url; + url += "catalog/release/"; + url += catalog_id; + + ok = curl.download(url, &out); + if (!ok) + { + if (nullptr != log) + { + *log << "Could not download json" << endl; + *log << "CURL:" << curl.get_error() << endl; + } + return result; + } + + out >> result; + return result; +} + +bool Monstercat_DL::download_cover(const string& catalog_id, const string& path) +{ + CURL_DL& curl = CURL_DL::get_handle(); + bool ok; + string url; + stringstream out; + ofstream out_file; + map out_headers; + string filename; + + url = "https://www.monstercat.com/release/"; + url += catalog_id; + url += "/cover"; + + ok = curl.download(url, &out, nullptr, &out_headers); + if (!ok) + { + if (nullptr != log) + { + *log << "Could not download image" << endl; + *log << "CURL:" << curl.get_error() << endl; + } + return false; + } + + if (0 == out_headers.count("content-type")) + { + if (nullptr != log) + { + *log << "Unknown content-type" << endl; + } + return false; + } + + filename = path; + if (out_headers["content-type"] == "image/jpeg") + { + filename += ".jpg"; + } + else if (out_headers["content-type"] == "image/png") + { + filename += ".png"; + } + + out_file.open(filename, std::ios::binary); + if (!out_file.is_open()) + { + if (nullptr != log) + { + *log << "Could not open file for write" << endl; + *log << filename << endl; + } + return false; + } + + out_file << out.rdbuf(); + out_file.close(); + + return true; +} diff --git a/monstercat_dl.h b/monstercat_dl.h new file mode 100644 index 0000000..01b5357 --- /dev/null +++ b/monstercat_dl.h @@ -0,0 +1,28 @@ +#ifndef MONSTERCAT_DL_H_ +#define MONSTERCAT_DL_H_ + +#include +#include +#include "json.hpp" + +class Monstercat_DL +{ +private: + std::ostream *log; + + std::string m_base_url; + + bool m_is_logged_in; + +public: + Monstercat_DL(); + ~Monstercat_DL(); + + bool login(const std::string& user, const std::string& pass); + bool logout(); + + nlohmann::json get_release_json(const std::string& catalog_id); + bool download_cover(const std::string& catalog_id, const std::string& path); +}; + +#endif // MONSTERCAT_DL_H_