diff --git a/CHANGELOG.md b/CHANGELOG.md index 89557bb..6492045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,3 @@ -## 0.3.4 (2017-05-10) -- sync dlcs (fixes #12) -- break content deserializing and sync code into smaller pieces (fixes #9) - ## 0.3.3 (2017-05-09) - normalize content directory names (fixes #11) diff --git a/Cargo.lock b/Cargo.lock index d1c7ebe..a6fb3ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,3 +1,20 @@ +[root] +name = "gog-sync" +version = "0.3.3" +dependencies = [ + "chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.24.1 (registry+https://github.com/rust-lang/crates.io-index)", + "curl 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "aho-corasick" version = "0.5.3" @@ -111,23 +128,6 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "gog-sync" -version = "0.3.4" -dependencies = [ - "chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.24.1 (registry+https://github.com/rust-lang/crates.io-index)", - "curl 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "idna" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index ca4a724..ea3391c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "gog-sync" -version = "0.3.4" +version = "0.3.3" authors = ["Sebastian Hugentobler "] description = "Synchronizes a GOG library with a local folder." documentation = "https://docs.rs/crate/gog-sync" -repository = "https://code.vanwa.ch/sebastian@vanwa.ch/gog-sync" +repository = "https://gitlab.com/thallian/gog-sync" readme = "README.md" keywords = ["gog", "sync", "cli"] categories = ["command-line-utilities"] diff --git a/README.md b/README.md index cb379be..56227a4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,14 @@ -# UNMAINTAINED AND BROKEN - -Used to work, does not anymore, maybe can be still of interest for someone - # GOG-SYNC - ## **Please note that this is alpha software, you should not trust it.** A small tool to synchronize the stuff in a [GOG](https://www.gog.com/) library with a local folder. -It builds on the work of the -[unofficial GOG API Documentation](https://gogapidocs.readthedocs.io/en/latest/). +It builds on the work of the [unofficial GOG API Documentation](https://gogapidocs.readthedocs.io/en/latest/). This is the first time I am building something with rust, so beware :) # Installation - Install from [crates.io](https://crates.io/crates/gog-sync). ``` @@ -23,9 +16,8 @@ cargo install gog-sync ``` # Configuration - -The configuration file is in the config folder as described by the xdg -specification with a prefix of `gog-sync`. +The configuration file is in the config folder as described by the xdg specification +with a prefix of `gog-sync`. For example on macOS or Linux @@ -41,38 +33,32 @@ A bare configuration with default values before first use: "movieStorage": ".", "osFilters": [], "languageFilters": [], - "resolutionFilters": [], "skipMovies": false, "skipGames": false } ``` -- _gameStorage_: Where to save games -- _movieStorage_: Where to save movies -- _osFilters_: An array of operating systems. If it is not empty, game data is - limited to the ones in the list. -- _languageFilters_: An array of languages. If it is not empty, game data is - limited to the ones in the list. -- _resolutionFilters_: An array of resolutions. If it is not empty, movie data - is limited to the ones in the list. -- _skipMovies_: Whether to skip movie content -- _skipGames_: Whether to skip game content +- *gameStorage*: Where to save games +- *movieStorage*: Where to save movies +- *osFilters*: An array of operating systems. If it is not empty, game data is limited to the ones in the list. +- *languageFilters*: An array of languages. If it is not empty, game data is limited to the ones in the list. +- *resolutionFilters*: An array of resolutions. If it is not empty, movie data is limited to the ones in the list. +- *skipMovies*: Whether to skip movie content +- *skipGames*: Whether to skip game content -Valid values for _osFilter_: - -- `linux` -- `mac` -- `windows` +Valid values for *osFilter*: +- `linux` +- `mac` +- `windows` Check on [gog.com](https://www.gog.com/) which languages are available. An incomplete list of resolutions on gog: - -- `DVD` -- `576p` -- `720p` -- `1080p` -- `4k` +- `DVD` +- `576p` +- `720p` +- `1080p` +- `4k` # Usage @@ -103,8 +89,8 @@ OPTIONS: gog-sync ``` -Normal invocation, uses the current working directory as storage if not -configured otherwise. +Normal invocation, uses the current working directory as storage if not configured +otherwise. --- diff --git a/src/gog.rs b/src/gog.rs index 8137c55..5dc69b0 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -15,18 +15,17 @@ use http::{Http, HttpError}; use models; use models::content::Content; use models::contentinfo::ContentInfo; -use models::data::Data; use models::datainfo::DataInfo; -use models::extra::Extra; use models::extrainfo::ExtraInfo; +use models::data::Data; use models::token::Token; use serde_json; use serde_json::Value; -use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::BTreeMap; +use std::fmt; use std::fs; use std::fs::File; -use std::fmt; use std::io; use std::io::Write; use std::path::Path; @@ -134,7 +133,7 @@ impl<'a> Gog<'a> { /// Syncs the contents of a gog account with a local folder. /// Uses a hash to figure out whether something has changed. pub fn sync(&mut self, - storage_path_games: &str, + storage_path: &str, storage_path_movies: &str, os_filters: &Vec, language_filters: &Vec, @@ -156,200 +155,120 @@ impl<'a> Gog<'a> { } }; + let content_root = if content.is_movie { + Path::new(storage_path_movies).join(&content.title) + } else { + Path::new(storage_path).join(&content.title) + }; + + let content_info_path = Path::new(&content_root).join("info.json"); + + let content_info_saved = + match ConfigFiles::load_from_path::(&content_info_path) { + Ok(value) => value, + Err(_) => ContentInfo::new(), + }; + if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { info!("filtering {}", content.title); continue; } - match self.sync_content(content, storage_path_games, storage_path_movies) { - Ok(_) => (), - Err(error) => { - warn!("{:?}", error); - continue; - } + let content_hash = models::get_hash(&content); + + if content_info_saved.hash == content_hash { + info!("{} already up to date.", &content.title); + continue; } - } - Ok(()) - } - fn sync_content(&mut self, - content: Content, - storage_path_games: &str, - storage_path_movies: &str) - -> Result<(), GogError> { + fs::create_dir_all(&content_root)?; - let content_root = if content.is_movie { - Path::new(storage_path_movies).join(&content.title) - } else { - Path::new(storage_path_games).join(&content.title) - }; - - let content_info_path = Path::new(&content_root).join("info.json"); - let content_info_saved = - match ConfigFiles::load_from_path::(&content_info_path) { - Ok(value) => value, - Err(_) => ContentInfo::new(), + let mut content_info = ContentInfo { + hash: content_hash, + id: content_id, + title: content.title.clone(), + cd_keys: content.cd_keys.clone(), + data: HashMap::new(), + extras: HashMap::new(), }; - let content_hash = models::get_hash(&content); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; - if content_info_saved.hash == content_hash { - info!("{} already up to date.", &content.title); - return Ok(()); - } + let key_root = content_root.join("keys"); - fs::create_dir_all(&content_root)?; + if content.cd_keys.len() > 0 { + fs::create_dir_all(&key_root)?; + } - let mut content_info = ContentInfo { - hash: content_hash, - id: content.id, - title: content.title.clone(), - cd_keys: content.cd_keys.clone(), - data: HashMap::new(), - extras: HashMap::new(), - }; + for (key, value) in content.cd_keys { + let key_path = key_root.join(format!("{}.txt", key)); + let mut key_file = File::create(&key_path)?; + key_file.write_all(value.as_bytes())? + } - ConfigFiles::save_to_path(&content_info_path, &content_info)?; + for data in content.data { + let data_hash_saved = match content_info_saved.data.get(&content_id.to_string()) { + Some(value) => value.hash, + None => u64::min_value(), + }; - self.save_keys(content.cd_keys, &content_root)?; + let data_hash = models::get_hash(&data); - for data in content.data { - self.save_data(content.title.as_str(), - data, - &content_root, - &content_info_path, - &content_info_saved, - &mut content_info)?; - } + if data_hash_saved == data_hash { + info!("{} already up to date.", &data.manual_url); + continue; + } - let dlc_root = Path::new(&storage_path_games) - .join(&content.title) - .join("dlcs"); + let data_root = Path::new(&content_root).join(&data.language); + fs::create_dir_all(&data_root)?; - for dlc in content.dlcs { - self.sync_content(dlc, - dlc_root.to_string_lossy().as_ref(), - storage_path_movies)?; - } + let data_uri = format!("https://embed.gog.com{}", data.manual_url); - for extra in content.extras { - self.save_extra(content.title.as_str(), - extra, - &content_root, - &content_info_path, - &content_info_saved, - &mut content_info)?; - } + info!("downloading {} for {}...", &data.manual_url, &content.title); + let filename = self.api_request_download(data_uri.as_str(), &data_root)?; - Ok(()) - } + let data_info = DataInfo { + hash: data_hash, + filename: filename, + language: data.language, + }; - fn save_file(&mut self, - content_title: &str, - relative_uri: &str, - data_root: &PathBuf) - -> Result { - fs::create_dir_all(&data_root)?; - let data_uri = format!("https://embed.gog.com{}", relative_uri); + content_info + .data + .insert(data.manual_url.clone(), data_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + } - let filename = self.api_request_get_filename(&data_uri)?; - let file_path = Path::new(&data_root).join(&filename); + for extra in content.extras { + let extra_hash_saved = + match content_info_saved.extras.get(&content_id.to_string()) { + Some(value) => value.hash, + None => u64::min_value(), + }; - if file_path.exists() { - info!("{} for {} already exists", &filename, content_title); - return Ok(filename.to_owned()); - } + let extra_hash = models::get_hash(&extra); - info!("downloading {} for {}...", &relative_uri, content_title); - self.api_request_download(data_uri.as_str(), &data_root) - } + if extra_hash_saved == extra_hash { + info!("{} already up to date.", &extra.manual_url); + continue; + } - fn save_data(&mut self, - content_title: &str, - data: Data, - content_root: &PathBuf, - content_info_path: &PathBuf, - content_info_saved: &ContentInfo, - content_info: &mut ContentInfo) - -> Result<(), GogError> { - let hash_saved = match content_info_saved.data.get(data.manual_url.as_str()) { - Some(value) => value.hash, - None => u64::min_value(), - }; + let extra_uri = format!("https://embed.gog.com{}", extra.manual_url); - let hash = models::get_hash(&data); - if hash_saved == hash { - info!("{} already up to date.", &data.manual_url); - return Ok(()); - } + info!("downloading {} for {}...", &extra.name, &content.title); + let filename = self.api_request_download(extra_uri.as_str(), &content_root)?; - let data_root = Path::new(&content_root).join(&data.language); + let extra_info = ExtraInfo { + hash: extra_hash, + filename: filename, + name: extra.name, + }; - let filename = self.save_file(content_title, &data.manual_url, &data_root)?; - - let data_info = DataInfo { - hash: hash, - filename: filename, - language: data.language, - }; - - content_info - .data - .insert(data.manual_url.clone(), data_info); - ConfigFiles::save_to_path(&content_info_path, &content_info)?; - - Ok(()) - } - - fn save_extra(&mut self, - content_title: &str, - extra: Extra, - content_root: &PathBuf, - content_info_path: &PathBuf, - content_info_saved: &ContentInfo, - content_info: &mut ContentInfo) - -> Result<(), GogError> { - let hash_saved = match content_info_saved.data.get(extra.manual_url.as_str()) { - Some(value) => value.hash, - None => u64::min_value(), - }; - - let hash = models::get_hash(&extra); - if hash_saved == hash { - info!("{} already up to date.", &extra.manual_url); - return Ok(()); - } - - let filename = self.save_file(content_title, &extra.manual_url, &content_root)?; - - let extra_info = ExtraInfo { - hash: hash, - filename: filename, - name: extra.name, - }; - - content_info - .extras - .insert(extra.manual_url.clone(), extra_info); - ConfigFiles::save_to_path(&content_info_path, &content_info)?; - - Ok(()) - } - - fn save_keys(&mut self, - cd_keys: BTreeMap, - content_root: &PathBuf) - -> Result<(), GogError> { - let key_root = content_root.join("keys"); - - if cd_keys.len() > 0 { - fs::create_dir_all(&key_root)?; - } - - for (key, value) in cd_keys { - let key_path = key_root.join(format!("{}.txt", key)); - let mut key_file = File::create(&key_path)?; - key_file.write_all(value.as_bytes())? + content_info + .extras + .insert(extra.manual_url.clone(), extra_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + } } Ok(()) @@ -430,7 +349,45 @@ impl<'a> Gog<'a> { content_ids.push(content_id_parsed); } - Ok(content_ids) + return Ok(content_ids); + } + + fn parse_cd_keys(&self, content_title: &str, raw_cd_keys: &str) -> BTreeMap { + let mut cd_keys = BTreeMap::new(); + let mut raw_cd_keys_fix = raw_cd_keys.to_owned(); + + if raw_cd_keys_fix.contains("") { + raw_cd_keys_fix = raw_cd_keys_fix + .replace("", ":") + .replace("", "") + .replace("", ""); + } + + raw_cd_keys_fix = raw_cd_keys_fix.replace("
", ":").replace("::", ":"); + + if raw_cd_keys_fix.contains(":") { + let splitted_keys = raw_cd_keys_fix.split(":"); + + let mut key_names: Vec = Vec::new(); + let mut key_values: Vec = Vec::new(); + + for (token_index, token) in splitted_keys.enumerate() { + if token_index % 2 == 0 { + key_names.push(token.to_owned()); + } else { + key_values.push(token.trim().to_owned()); + } + } + + for (index, key_name) in key_names.iter().enumerate() { + let key_value = key_values[index].clone(); + cd_keys.insert(key_name.clone(), key_value); + } + } else if !raw_cd_keys_fix.is_empty() { + cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned()); + } + + return cd_keys; } fn api_request_ensure_token(&mut self, response: &str) -> Result { @@ -459,15 +416,6 @@ impl<'a> Gog<'a> { } } - fn api_request_get_filename(&mut self, uri: &str) -> Result { - let response = self.http_client.get_filename(uri)?; - if self.api_request_ensure_token(&response)? { - Ok(self.http_client.get_filename(uri)?) - } else { - Ok(response) - } - } - fn api_request_download(&mut self, uri: &str, path: &PathBuf) -> Result { let response = self.http_client.download(uri, path)?; if self.api_request_ensure_token(&response)? { @@ -488,11 +436,118 @@ impl<'a> Gog<'a> { let response = self.api_request_get(content_uri.as_str())?; - models::content::deserialize(content_id, - response.as_str(), - os_filters, - language_filters, - resolution_filters) + let content_raw: Value = serde_json::from_str(response.as_str())?; + debug!("found {:?}", &content_raw); + + let mut content: Content = serde_json::from_str(&response)?; + content.id = content_id; + + let downloads = &content_raw["downloads"]; + + if content_raw.is_object() && + !content_raw + .as_object() + .unwrap() + .contains_key("forumLink") { + return Err(GogError::Error("No forumLink property")); + } + + let is_movie = match content_raw["forumLink"].as_str() { + Some(value) => value == "https://embed.gog.com/forum/movies", + None => false, + }; + + content.is_movie = is_movie; + + if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") { + let cd_keys_raw = content_raw["cdKey"].as_str().unwrap(); + content.cd_keys = self.parse_cd_keys(&content.title, cd_keys_raw); + } + + debug!("processing installer fields: {:?}", &downloads); + for languages in downloads.as_array() { + for language in languages { + if !language.is_array() || language.as_array().unwrap().len() < 2 { + error!("Skipping a language for {}", content.title); + continue; + } + + let data_language = match language[0].as_str() { + Some(value) => value.to_lowercase(), + None => String::default(), + }; + + if data_language.is_empty() { + error!("Skipping a language for {}", content.title); + continue; + } + + if !is_movie && !language_filters.is_empty() && + !language_filters.contains(&data_language) { + info!("Skipping {} for {}", &data_language, content.title); + continue; + } + + for systems in language[1].as_object() { + for system in systems.keys() { + if is_movie && !os_filters.is_empty() && !os_filters.contains(system) { + info!("Skipping {} {} for {}", + &data_language, + system, + content.title); + continue; + } + + for real_downloads in systems.get(system) { + for real_download in real_downloads.as_array() { + for download in real_download { + if !download.is_object() || + !download.as_object().unwrap().contains_key("manualUrl") || + !download.as_object().unwrap().contains_key("name") { + error!("Skipping data for {}", content.title); + continue; + } + + let name: &str = download["name"].as_str().unwrap(); + + + if content.is_movie && !resolution_filters.is_empty() { + let mut found_resolution = false; + for resolution_filter in resolution_filters { + let filter = format!("({})", resolution_filter); + if name.ends_with(&filter) { + found_resolution = true; + break; + } + } + + if !found_resolution { + info!("Skipping {}: not a suitable resolution.", name); + continue; + } + } + + let data = Data { + manual_url: String::from(download["manualUrl"] + .as_str() + .unwrap()), + version: String::from(download["version"] + .as_str() + .unwrap_or("")), + os: system.clone(), + language: data_language.clone(), + }; + + content.data.push(data); + } + } + } + } + } + } + } + + Ok(content) } fn games_uri(&self) -> String { diff --git a/src/http.rs b/src/http.rs index c6b6f29..61335ff 100644 --- a/src/http.rs +++ b/src/http.rs @@ -96,11 +96,10 @@ impl Http { self.curl.url(uri)?; { let mut transfer = self.curl.transfer(); - transfer - .write_function(|new_data| { - data.extend_from_slice(new_data); - Ok(new_data.len()) - })?; + transfer.write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + })?; transfer.perform()?; } @@ -130,46 +129,6 @@ impl Http { } } - /// Find the filename for a download uri. - /// - /// Useful if the initial link gets redirected. - /// - /// The filename is taken from the last URI segment and returned in the result. - /// # Example - /// ``` - /// use http::Http; - /// use std::path::Path; - /// - /// let mut http_client = Http::new(); - /// - /// http_client.get_filename("https://example.com/sed"); - /// ``` - pub fn get_filename(&mut self, download_uri: &str) -> Result { - self.curl.url(download_uri)?; - self.curl.nobody(true)?; - self.curl.perform()?; - self.curl.nobody(false)?; - - let download_url_string = match self.curl.effective_url()? { - Some(value) => value, - None => return Err(HttpError::Error("Can't get effective download url.")), - }; - - let download_url = Url::parse(download_url_string)?; - - let download_url_segments = match download_url.path_segments() { - Some(value) => value, - None => return Err(HttpError::Error("Can't parse download segments.")), - }; - - let file_name = match download_url_segments.last() { - Some(value) => value, - None => return Err(HttpError::Error("No segments in download url.")), - }; - - Ok(file_name.to_owned()) - } - /// Download a file to the specified folder without creating the folder. /// /// The filename is taken from the last URI segment and returned in the result. @@ -193,14 +152,13 @@ impl Http { self.curl.url(download_uri)?; { let mut transfer = self.curl.transfer(); - transfer - .write_function(|data| { - match file_download.write(data) { - Ok(_) => Ok(()), - Err(error) => Err(HttpError::IOError(error)), - }?; - Ok(data.len()) - })?; + transfer.write_function(|data| { + match file_download.write(data) { + Ok(_) => Ok(()), + Err(error) => Err(HttpError::IOError(error)), + }?; + Ok(data.len()) + })?; transfer.perform()?; } diff --git a/src/main.rs b/src/main.rs index 10d7b7d..0c0bcd9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,7 +29,7 @@ fn main() { env_logger::init().unwrap(); let matches = App::new("Gog Synchronizer") - .version("0.3.4") + .version("0.3.3") .author("Sebastian Hugentobler ") .about("Synchronizes your gog library to a local folder.") .arg(Arg::with_name("game-storage") diff --git a/src/models/content.rs b/src/models/content.rs index 7e47df2..a311e21 100644 --- a/src/models/content.rs +++ b/src/models/content.rs @@ -1,10 +1,7 @@ -use gog::GogError; use models::data::Data; use models::extra::Extra; use regex::Regex; use serde::{Deserialize, Deserializer}; -use serde_json; -use serde_json::Value; use std::collections::BTreeMap; use std::fmt; @@ -14,7 +11,7 @@ use std::fmt; pub struct Content { #[serde(skip_deserializing)] pub id: u64, - #[serde(deserialize_with = "deserialize_title")] + #[serde(deserialize_with = "normalize_title")] pub title: String, #[serde(skip_deserializing)] #[serde(rename(deserialize = "cdKey"))] @@ -24,7 +21,6 @@ pub struct Content { #[serde(skip_deserializing)] pub data: Vec, pub extras: Vec, - pub dlcs: Vec, } impl fmt::Display for Content { @@ -33,7 +29,7 @@ impl fmt::Display for Content { } } -fn deserialize_title(deserializer: D) -> Result +fn normalize_title(deserializer: D) -> Result where D: Deserializer { let raw_title = String::deserialize(deserializer)?.replace(":", " - "); @@ -51,213 +47,3 @@ fn deserialize_title(deserializer: D) -> Result Ok(title_whitespace) } - -/// Keys come in at least three different forms. -/// It is possible to parse them all into the same structure, a map. -/// -/// # Variants -/// ## I -/// Only one key and the value is the key itself. -/// `1234-5678-1234-5678` -/// -/// ## II -/// Multiple keys split at single `
` tags with the key name and value itself -/// split at `:
`. -/// ``` -/// Neverwinter Nights:
1234-5678-1234-5678
Shadows of Undrentide:
1234-5678-1234-5678
Hordes of the Underdark:
1234-5678-1234-5678 -/// ``` -/// -/// ## III -/// Multiple keys split at `` tags with the key name and value itself -/// again split at `:
`. -/// -/// ``` -/// Neverwinter Nights 2:
1234-5678-1234-5678Mask of the Betrayer:
1234-5678-1234-5678Storm of Zehir:
1234-5678-1234-5678 -/// ``` -/// -/// With this information the method works as follows: -/// - replace every `` with `:` -/// - remove every `` and `` -/// - replace every `
` with `:` -/// - replace every `::` with `:` -/// - if there is at least one `:` -/// - split at `:` -/// - every odd position in the resulting array is a key name, every even its corresponding value -/// - else -/// - the content name is the key name and the value is used as is -/// -fn deserialize_cd_keys(content_title: &str, raw_cd_keys: &str) -> BTreeMap { - let mut cd_keys = BTreeMap::new(); - let mut raw_cd_keys_fix = raw_cd_keys.to_owned(); - - if raw_cd_keys_fix.contains("") { - raw_cd_keys_fix = raw_cd_keys_fix - .replace("", ":") - .replace("", "") - .replace("", ""); - } - - raw_cd_keys_fix = raw_cd_keys_fix.replace("
", ":").replace("::", ":"); - - if raw_cd_keys_fix.contains(":") { - let splitted_keys = raw_cd_keys_fix.split(":"); - - let mut key_names: Vec = Vec::new(); - let mut key_values: Vec = Vec::new(); - - for (token_index, token) in splitted_keys.enumerate() { - if token_index % 2 == 0 { - key_names.push(token.to_owned()); - } else { - key_values.push(token.trim().to_owned()); - } - } - - for (index, key_name) in key_names.iter().enumerate() { - let key_value = key_values[index].clone(); - cd_keys.insert(key_name.clone(), key_value); - } - } else if !raw_cd_keys_fix.is_empty() { - cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned()); - } - - cd_keys -} - -pub fn deserialize(content_id: u64, - content_string: &str, - os_filters: &Vec, - language_filters: &Vec, - resolution_filters: &Vec) - -> Result { - let content_raw: Value = serde_json::from_str(content_string)?; - debug!("found {:?}", &content_raw); - - let mut content: Content = serde_json::from_str(content_string)?; - content.id = content_id; - - let dlcs = &content_raw["dlcs"]; - let downloads = &content_raw["downloads"]; - - if content_raw.is_object() && - !content_raw - .as_object() - .unwrap() - .contains_key("forumLink") { - return Err(GogError::Error("No forumLink property")); - } - - let is_movie = match content_raw["forumLink"].as_str() { - Some(value) => value == "https://embed.gog.com/forum/movies", - None => false, - }; - - content.is_movie = is_movie; - - if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") { - let cd_keys_raw = content_raw["cdKey"].as_str().unwrap(); - content.cd_keys = deserialize_cd_keys(&content.title, cd_keys_raw); - } - - match dlcs.as_array() { - Some(value) => { - if value.len() > 0 { - debug!("processing dlc fields: {:?}", &value); - for dlc in value { - let dlc = deserialize(content_id, - serde_json::to_string(&dlc).unwrap().as_str(), - os_filters, - language_filters, - resolution_filters)?; - content.dlcs.push(dlc); - } - } - } - None => (), - } - - - debug!("processing installer fields: {:?}", &downloads); - for languages in downloads.as_array() { - for language in languages { - if !language.is_array() || language.as_array().unwrap().len() < 2 { - error!("Skipping a language for {}", content.title); - continue; - } - - let data_language = match language[0].as_str() { - Some(value) => value.to_lowercase(), - None => String::default(), - }; - - if data_language.is_empty() { - error!("Skipping a language for {}", content.title); - continue; - } - - if !is_movie && !language_filters.is_empty() && - !language_filters.contains(&data_language) { - info!("Skipping {} for {}", &data_language, content.title); - continue; - } - - for systems in language[1].as_object() { - for system in systems.keys() { - if is_movie && !os_filters.is_empty() && !os_filters.contains(system) { - info!("Skipping {} {} for {}", - &data_language, - system, - content.title); - continue; - } - - for real_downloads in systems.get(system) { - for real_download in real_downloads.as_array() { - for download in real_download { - if !download.is_object() || - !download.as_object().unwrap().contains_key("manualUrl") || - !download.as_object().unwrap().contains_key("name") { - error!("Skipping data for {}", content.title); - continue; - } - - let name: &str = download["name"].as_str().unwrap(); - - - if content.is_movie && !resolution_filters.is_empty() { - let mut found_resolution = false; - for resolution_filter in resolution_filters { - let filter = format!("({})", resolution_filter); - if name.ends_with(&filter) { - found_resolution = true; - break; - } - } - - if !found_resolution { - info!("Skipping {}: not a suitable resolution.", name); - continue; - } - } - - let data = Data { - manual_url: String::from(download["manualUrl"] - .as_str() - .unwrap()), - version: - String::from(download["version"].as_str().unwrap_or("")), - os: system.clone(), - language: data_language.clone(), - }; - - content.data.push(data); - } - } - } - } - } - } - } - - Ok(content) -}