diff --git a/CHANGELOG.md b/CHANGELOG.md index 6492045..89557bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.3.4 (2017-05-10) +- sync dlcs (fixes #12) +- break content deserializing and sync code into smaller pieces (fixes #9) + ## 0.3.3 (2017-05-09) - normalize content directory names (fixes #11) diff --git a/Cargo.lock b/Cargo.lock index a6fb3ef..d1c7ebe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,20 +1,3 @@ -[root] -name = "gog-sync" -version = "0.3.3" -dependencies = [ - "chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.24.1 (registry+https://github.com/rust-lang/crates.io-index)", - "curl 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "aho-corasick" version = "0.5.3" @@ -128,6 +111,23 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "gog-sync" +version = "0.3.4" +dependencies = [ + "chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.24.1 (registry+https://github.com/rust-lang/crates.io-index)", + "curl 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "idna" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index ea3391c..ca4a724 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "gog-sync" -version = "0.3.3" +version = "0.3.4" authors = ["Sebastian Hugentobler "] description = "Synchronizes a GOG library with a local folder." documentation = "https://docs.rs/crate/gog-sync" -repository = "https://gitlab.com/thallian/gog-sync" +repository = "https://code.vanwa.ch/sebastian@vanwa.ch/gog-sync" readme = "README.md" keywords = ["gog", "sync", "cli"] categories = ["command-line-utilities"] diff --git a/README.md b/README.md index 56227a4..cb379be 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,21 @@ +# UNMAINTAINED AND BROKEN + +Used to work, does not anymore, maybe can be still of interest for someone + # GOG-SYNC + ## **Please note that this is alpha software, you should not trust it.** A small tool to synchronize the stuff in a [GOG](https://www.gog.com/) library with a local folder. -It builds on the work of the [unofficial GOG API Documentation](https://gogapidocs.readthedocs.io/en/latest/). +It builds on the work of the +[unofficial GOG API Documentation](https://gogapidocs.readthedocs.io/en/latest/). This is the first time I am building something with rust, so beware :) # Installation + Install from [crates.io](https://crates.io/crates/gog-sync). ``` @@ -16,8 +23,9 @@ cargo install gog-sync ``` # Configuration -The configuration file is in the config folder as described by the xdg specification -with a prefix of `gog-sync`. + +The configuration file is in the config folder as described by the xdg +specification with a prefix of `gog-sync`. For example on macOS or Linux @@ -33,32 +41,38 @@ A bare configuration with default values before first use: "movieStorage": ".", "osFilters": [], "languageFilters": [], + "resolutionFilters": [], "skipMovies": false, "skipGames": false } ``` -- *gameStorage*: Where to save games -- *movieStorage*: Where to save movies -- *osFilters*: An array of operating systems. If it is not empty, game data is limited to the ones in the list. -- *languageFilters*: An array of languages. If it is not empty, game data is limited to the ones in the list. -- *resolutionFilters*: An array of resolutions. If it is not empty, movie data is limited to the ones in the list. -- *skipMovies*: Whether to skip movie content -- *skipGames*: Whether to skip game content +- _gameStorage_: Where to save games +- _movieStorage_: Where to save movies +- _osFilters_: An array of operating systems. If it is not empty, game data is + limited to the ones in the list. +- _languageFilters_: An array of languages. If it is not empty, game data is + limited to the ones in the list. +- _resolutionFilters_: An array of resolutions. If it is not empty, movie data + is limited to the ones in the list. +- _skipMovies_: Whether to skip movie content +- _skipGames_: Whether to skip game content -Valid values for *osFilter*: -- `linux` -- `mac` -- `windows` +Valid values for _osFilter_: + +- `linux` +- `mac` +- `windows` Check on [gog.com](https://www.gog.com/) which languages are available. An incomplete list of resolutions on gog: -- `DVD` -- `576p` -- `720p` -- `1080p` -- `4k` + +- `DVD` +- `576p` +- `720p` +- `1080p` +- `4k` # Usage @@ -89,8 +103,8 @@ OPTIONS: gog-sync ``` -Normal invocation, uses the current working directory as storage if not configured -otherwise. +Normal invocation, uses the current working directory as storage if not +configured otherwise. --- diff --git a/src/gog.rs b/src/gog.rs index 5dc69b0..8137c55 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -15,17 +15,18 @@ use http::{Http, HttpError}; use models; use models::content::Content; use models::contentinfo::ContentInfo; -use models::datainfo::DataInfo; -use models::extrainfo::ExtraInfo; use models::data::Data; +use models::datainfo::DataInfo; +use models::extra::Extra; +use models::extrainfo::ExtraInfo; use models::token::Token; use serde_json; use serde_json::Value; -use std::collections::HashMap; use std::collections::BTreeMap; -use std::fmt; +use std::collections::HashMap; use std::fs; use std::fs::File; +use std::fmt; use std::io; use std::io::Write; use std::path::Path; @@ -133,7 +134,7 @@ impl<'a> Gog<'a> { /// Syncs the contents of a gog account with a local folder. /// Uses a hash to figure out whether something has changed. pub fn sync(&mut self, - storage_path: &str, + storage_path_games: &str, storage_path_movies: &str, os_filters: &Vec, language_filters: &Vec, @@ -155,120 +156,200 @@ impl<'a> Gog<'a> { } }; - let content_root = if content.is_movie { - Path::new(storage_path_movies).join(&content.title) - } else { - Path::new(storage_path).join(&content.title) - }; - - let content_info_path = Path::new(&content_root).join("info.json"); - - let content_info_saved = - match ConfigFiles::load_from_path::(&content_info_path) { - Ok(value) => value, - Err(_) => ContentInfo::new(), - }; - if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { info!("filtering {}", content.title); continue; } - let content_hash = models::get_hash(&content); - - if content_info_saved.hash == content_hash { - info!("{} already up to date.", &content.title); - continue; + match self.sync_content(content, storage_path_games, storage_path_movies) { + Ok(_) => (), + Err(error) => { + warn!("{:?}", error); + continue; + } } + } + Ok(()) + } - fs::create_dir_all(&content_root)?; + fn sync_content(&mut self, + content: Content, + storage_path_games: &str, + storage_path_movies: &str) + -> Result<(), GogError> { - let mut content_info = ContentInfo { - hash: content_hash, - id: content_id, - title: content.title.clone(), - cd_keys: content.cd_keys.clone(), - data: HashMap::new(), - extras: HashMap::new(), + let content_root = if content.is_movie { + Path::new(storage_path_movies).join(&content.title) + } else { + Path::new(storage_path_games).join(&content.title) + }; + + let content_info_path = Path::new(&content_root).join("info.json"); + let content_info_saved = + match ConfigFiles::load_from_path::(&content_info_path) { + Ok(value) => value, + Err(_) => ContentInfo::new(), }; - ConfigFiles::save_to_path(&content_info_path, &content_info)?; + let content_hash = models::get_hash(&content); - let key_root = content_root.join("keys"); + if content_info_saved.hash == content_hash { + info!("{} already up to date.", &content.title); + return Ok(()); + } - if content.cd_keys.len() > 0 { - fs::create_dir_all(&key_root)?; - } + fs::create_dir_all(&content_root)?; - for (key, value) in content.cd_keys { - let key_path = key_root.join(format!("{}.txt", key)); - let mut key_file = File::create(&key_path)?; - key_file.write_all(value.as_bytes())? - } + let mut content_info = ContentInfo { + hash: content_hash, + id: content.id, + title: content.title.clone(), + cd_keys: content.cd_keys.clone(), + data: HashMap::new(), + extras: HashMap::new(), + }; - for data in content.data { - let data_hash_saved = match content_info_saved.data.get(&content_id.to_string()) { - Some(value) => value.hash, - None => u64::min_value(), - }; + ConfigFiles::save_to_path(&content_info_path, &content_info)?; - let data_hash = models::get_hash(&data); + self.save_keys(content.cd_keys, &content_root)?; - if data_hash_saved == data_hash { - info!("{} already up to date.", &data.manual_url); - continue; - } + for data in content.data { + self.save_data(content.title.as_str(), + data, + &content_root, + &content_info_path, + &content_info_saved, + &mut content_info)?; + } - let data_root = Path::new(&content_root).join(&data.language); - fs::create_dir_all(&data_root)?; + let dlc_root = Path::new(&storage_path_games) + .join(&content.title) + .join("dlcs"); - let data_uri = format!("https://embed.gog.com{}", data.manual_url); + for dlc in content.dlcs { + self.sync_content(dlc, + dlc_root.to_string_lossy().as_ref(), + storage_path_movies)?; + } - info!("downloading {} for {}...", &data.manual_url, &content.title); - let filename = self.api_request_download(data_uri.as_str(), &data_root)?; + for extra in content.extras { + self.save_extra(content.title.as_str(), + extra, + &content_root, + &content_info_path, + &content_info_saved, + &mut content_info)?; + } - let data_info = DataInfo { - hash: data_hash, - filename: filename, - language: data.language, - }; + Ok(()) + } - content_info - .data - .insert(data.manual_url.clone(), data_info); - ConfigFiles::save_to_path(&content_info_path, &content_info)?; - } + fn save_file(&mut self, + content_title: &str, + relative_uri: &str, + data_root: &PathBuf) + -> Result { + fs::create_dir_all(&data_root)?; + let data_uri = format!("https://embed.gog.com{}", relative_uri); - for extra in content.extras { - let extra_hash_saved = - match content_info_saved.extras.get(&content_id.to_string()) { - Some(value) => value.hash, - None => u64::min_value(), - }; + let filename = self.api_request_get_filename(&data_uri)?; + let file_path = Path::new(&data_root).join(&filename); - let extra_hash = models::get_hash(&extra); + if file_path.exists() { + info!("{} for {} already exists", &filename, content_title); + return Ok(filename.to_owned()); + } - if extra_hash_saved == extra_hash { - info!("{} already up to date.", &extra.manual_url); - continue; - } + info!("downloading {} for {}...", &relative_uri, content_title); + self.api_request_download(data_uri.as_str(), &data_root) + } - let extra_uri = format!("https://embed.gog.com{}", extra.manual_url); + fn save_data(&mut self, + content_title: &str, + data: Data, + content_root: &PathBuf, + content_info_path: &PathBuf, + content_info_saved: &ContentInfo, + content_info: &mut ContentInfo) + -> Result<(), GogError> { + let hash_saved = match content_info_saved.data.get(data.manual_url.as_str()) { + Some(value) => value.hash, + None => u64::min_value(), + }; - info!("downloading {} for {}...", &extra.name, &content.title); - let filename = self.api_request_download(extra_uri.as_str(), &content_root)?; + let hash = models::get_hash(&data); + if hash_saved == hash { + info!("{} already up to date.", &data.manual_url); + return Ok(()); + } - let extra_info = ExtraInfo { - hash: extra_hash, - filename: filename, - name: extra.name, - }; + let data_root = Path::new(&content_root).join(&data.language); - content_info - .extras - .insert(extra.manual_url.clone(), extra_info); - ConfigFiles::save_to_path(&content_info_path, &content_info)?; - } + let filename = self.save_file(content_title, &data.manual_url, &data_root)?; + + let data_info = DataInfo { + hash: hash, + filename: filename, + language: data.language, + }; + + content_info + .data + .insert(data.manual_url.clone(), data_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + + Ok(()) + } + + fn save_extra(&mut self, + content_title: &str, + extra: Extra, + content_root: &PathBuf, + content_info_path: &PathBuf, + content_info_saved: &ContentInfo, + content_info: &mut ContentInfo) + -> Result<(), GogError> { + let hash_saved = match content_info_saved.data.get(extra.manual_url.as_str()) { + Some(value) => value.hash, + None => u64::min_value(), + }; + + let hash = models::get_hash(&extra); + if hash_saved == hash { + info!("{} already up to date.", &extra.manual_url); + return Ok(()); + } + + let filename = self.save_file(content_title, &extra.manual_url, &content_root)?; + + let extra_info = ExtraInfo { + hash: hash, + filename: filename, + name: extra.name, + }; + + content_info + .extras + .insert(extra.manual_url.clone(), extra_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + + Ok(()) + } + + fn save_keys(&mut self, + cd_keys: BTreeMap, + content_root: &PathBuf) + -> Result<(), GogError> { + let key_root = content_root.join("keys"); + + if cd_keys.len() > 0 { + fs::create_dir_all(&key_root)?; + } + + for (key, value) in cd_keys { + let key_path = key_root.join(format!("{}.txt", key)); + let mut key_file = File::create(&key_path)?; + key_file.write_all(value.as_bytes())? } Ok(()) @@ -349,45 +430,7 @@ impl<'a> Gog<'a> { content_ids.push(content_id_parsed); } - return Ok(content_ids); - } - - fn parse_cd_keys(&self, content_title: &str, raw_cd_keys: &str) -> BTreeMap { - let mut cd_keys = BTreeMap::new(); - let mut raw_cd_keys_fix = raw_cd_keys.to_owned(); - - if raw_cd_keys_fix.contains("") { - raw_cd_keys_fix = raw_cd_keys_fix - .replace("", ":") - .replace("", "") - .replace("", ""); - } - - raw_cd_keys_fix = raw_cd_keys_fix.replace("
", ":").replace("::", ":"); - - if raw_cd_keys_fix.contains(":") { - let splitted_keys = raw_cd_keys_fix.split(":"); - - let mut key_names: Vec = Vec::new(); - let mut key_values: Vec = Vec::new(); - - for (token_index, token) in splitted_keys.enumerate() { - if token_index % 2 == 0 { - key_names.push(token.to_owned()); - } else { - key_values.push(token.trim().to_owned()); - } - } - - for (index, key_name) in key_names.iter().enumerate() { - let key_value = key_values[index].clone(); - cd_keys.insert(key_name.clone(), key_value); - } - } else if !raw_cd_keys_fix.is_empty() { - cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned()); - } - - return cd_keys; + Ok(content_ids) } fn api_request_ensure_token(&mut self, response: &str) -> Result { @@ -416,6 +459,15 @@ impl<'a> Gog<'a> { } } + fn api_request_get_filename(&mut self, uri: &str) -> Result { + let response = self.http_client.get_filename(uri)?; + if self.api_request_ensure_token(&response)? { + Ok(self.http_client.get_filename(uri)?) + } else { + Ok(response) + } + } + fn api_request_download(&mut self, uri: &str, path: &PathBuf) -> Result { let response = self.http_client.download(uri, path)?; if self.api_request_ensure_token(&response)? { @@ -436,118 +488,11 @@ impl<'a> Gog<'a> { let response = self.api_request_get(content_uri.as_str())?; - let content_raw: Value = serde_json::from_str(response.as_str())?; - debug!("found {:?}", &content_raw); - - let mut content: Content = serde_json::from_str(&response)?; - content.id = content_id; - - let downloads = &content_raw["downloads"]; - - if content_raw.is_object() && - !content_raw - .as_object() - .unwrap() - .contains_key("forumLink") { - return Err(GogError::Error("No forumLink property")); - } - - let is_movie = match content_raw["forumLink"].as_str() { - Some(value) => value == "https://embed.gog.com/forum/movies", - None => false, - }; - - content.is_movie = is_movie; - - if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") { - let cd_keys_raw = content_raw["cdKey"].as_str().unwrap(); - content.cd_keys = self.parse_cd_keys(&content.title, cd_keys_raw); - } - - debug!("processing installer fields: {:?}", &downloads); - for languages in downloads.as_array() { - for language in languages { - if !language.is_array() || language.as_array().unwrap().len() < 2 { - error!("Skipping a language for {}", content.title); - continue; - } - - let data_language = match language[0].as_str() { - Some(value) => value.to_lowercase(), - None => String::default(), - }; - - if data_language.is_empty() { - error!("Skipping a language for {}", content.title); - continue; - } - - if !is_movie && !language_filters.is_empty() && - !language_filters.contains(&data_language) { - info!("Skipping {} for {}", &data_language, content.title); - continue; - } - - for systems in language[1].as_object() { - for system in systems.keys() { - if is_movie && !os_filters.is_empty() && !os_filters.contains(system) { - info!("Skipping {} {} for {}", - &data_language, - system, - content.title); - continue; - } - - for real_downloads in systems.get(system) { - for real_download in real_downloads.as_array() { - for download in real_download { - if !download.is_object() || - !download.as_object().unwrap().contains_key("manualUrl") || - !download.as_object().unwrap().contains_key("name") { - error!("Skipping data for {}", content.title); - continue; - } - - let name: &str = download["name"].as_str().unwrap(); - - - if content.is_movie && !resolution_filters.is_empty() { - let mut found_resolution = false; - for resolution_filter in resolution_filters { - let filter = format!("({})", resolution_filter); - if name.ends_with(&filter) { - found_resolution = true; - break; - } - } - - if !found_resolution { - info!("Skipping {}: not a suitable resolution.", name); - continue; - } - } - - let data = Data { - manual_url: String::from(download["manualUrl"] - .as_str() - .unwrap()), - version: String::from(download["version"] - .as_str() - .unwrap_or("")), - os: system.clone(), - language: data_language.clone(), - }; - - content.data.push(data); - } - } - } - } - } - } - } - - Ok(content) + models::content::deserialize(content_id, + response.as_str(), + os_filters, + language_filters, + resolution_filters) } fn games_uri(&self) -> String { diff --git a/src/http.rs b/src/http.rs index 61335ff..c6b6f29 100644 --- a/src/http.rs +++ b/src/http.rs @@ -96,10 +96,11 @@ impl Http { self.curl.url(uri)?; { let mut transfer = self.curl.transfer(); - transfer.write_function(|new_data| { - data.extend_from_slice(new_data); - Ok(new_data.len()) - })?; + transfer + .write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + })?; transfer.perform()?; } @@ -129,6 +130,46 @@ impl Http { } } + /// Find the filename for a download uri. + /// + /// Useful if the initial link gets redirected. + /// + /// The filename is taken from the last URI segment and returned in the result. + /// # Example + /// ``` + /// use http::Http; + /// use std::path::Path; + /// + /// let mut http_client = Http::new(); + /// + /// http_client.get_filename("https://example.com/sed"); + /// ``` + pub fn get_filename(&mut self, download_uri: &str) -> Result { + self.curl.url(download_uri)?; + self.curl.nobody(true)?; + self.curl.perform()?; + self.curl.nobody(false)?; + + let download_url_string = match self.curl.effective_url()? { + Some(value) => value, + None => return Err(HttpError::Error("Can't get effective download url.")), + }; + + let download_url = Url::parse(download_url_string)?; + + let download_url_segments = match download_url.path_segments() { + Some(value) => value, + None => return Err(HttpError::Error("Can't parse download segments.")), + }; + + let file_name = match download_url_segments.last() { + Some(value) => value, + None => return Err(HttpError::Error("No segments in download url.")), + }; + + Ok(file_name.to_owned()) + } + /// Download a file to the specified folder without creating the folder. /// /// The filename is taken from the last URI segment and returned in the result. @@ -152,13 +193,14 @@ impl Http { self.curl.url(download_uri)?; { let mut transfer = self.curl.transfer(); - transfer.write_function(|data| { - match file_download.write(data) { - Ok(_) => Ok(()), - Err(error) => Err(HttpError::IOError(error)), - }?; - Ok(data.len()) - })?; + transfer + .write_function(|data| { + match file_download.write(data) { + Ok(_) => Ok(()), + Err(error) => Err(HttpError::IOError(error)), + }?; + Ok(data.len()) + })?; transfer.perform()?; } diff --git a/src/main.rs b/src/main.rs index 0c0bcd9..10d7b7d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,7 +29,7 @@ fn main() { env_logger::init().unwrap(); let matches = App::new("Gog Synchronizer") - .version("0.3.3") + .version("0.3.4") .author("Sebastian Hugentobler ") .about("Synchronizes your gog library to a local folder.") .arg(Arg::with_name("game-storage") diff --git a/src/models/content.rs b/src/models/content.rs index a311e21..7e47df2 100644 --- a/src/models/content.rs +++ b/src/models/content.rs @@ -1,7 +1,10 @@ +use gog::GogError; use models::data::Data; use models::extra::Extra; use regex::Regex; use serde::{Deserialize, Deserializer}; +use serde_json; +use serde_json::Value; use std::collections::BTreeMap; use std::fmt; @@ -11,7 +14,7 @@ use std::fmt; pub struct Content { #[serde(skip_deserializing)] pub id: u64, - #[serde(deserialize_with = "normalize_title")] + #[serde(deserialize_with = "deserialize_title")] pub title: String, #[serde(skip_deserializing)] #[serde(rename(deserialize = "cdKey"))] @@ -21,6 +24,7 @@ pub struct Content { #[serde(skip_deserializing)] pub data: Vec, pub extras: Vec, + pub dlcs: Vec, } impl fmt::Display for Content { @@ -29,7 +33,7 @@ impl fmt::Display for Content { } } -fn normalize_title(deserializer: D) -> Result +fn deserialize_title(deserializer: D) -> Result where D: Deserializer { let raw_title = String::deserialize(deserializer)?.replace(":", " - "); @@ -47,3 +51,213 @@ fn normalize_title(deserializer: D) -> Result Ok(title_whitespace) } + +/// Keys come in at least three different forms. +/// It is possible to parse them all into the same structure, a map. +/// +/// # Variants +/// ## I +/// Only one key and the value is the key itself. +/// `1234-5678-1234-5678` +/// +/// ## II +/// Multiple keys split at single `
` tags with the key name and value itself +/// split at `:
`. +/// ``` +/// Neverwinter Nights:
1234-5678-1234-5678
Shadows of Undrentide:
1234-5678-1234-5678
Hordes of the Underdark:
1234-5678-1234-5678 +/// ``` +/// +/// ## III +/// Multiple keys split at `` tags with the key name and value itself +/// again split at `:
`. +/// +/// ``` +/// Neverwinter Nights 2:
1234-5678-1234-5678Mask of the Betrayer:
1234-5678-1234-5678Storm of Zehir:
1234-5678-1234-5678 +/// ``` +/// +/// With this information the method works as follows: +/// - replace every `` with `:` +/// - remove every `` and `` +/// - replace every `
` with `:` +/// - replace every `::` with `:` +/// - if there is at least one `:` +/// - split at `:` +/// - every odd position in the resulting array is a key name, every even its corresponding value +/// - else +/// - the content name is the key name and the value is used as is +/// +fn deserialize_cd_keys(content_title: &str, raw_cd_keys: &str) -> BTreeMap { + let mut cd_keys = BTreeMap::new(); + let mut raw_cd_keys_fix = raw_cd_keys.to_owned(); + + if raw_cd_keys_fix.contains("") { + raw_cd_keys_fix = raw_cd_keys_fix + .replace("", ":") + .replace("", "") + .replace("", ""); + } + + raw_cd_keys_fix = raw_cd_keys_fix.replace("
", ":").replace("::", ":"); + + if raw_cd_keys_fix.contains(":") { + let splitted_keys = raw_cd_keys_fix.split(":"); + + let mut key_names: Vec = Vec::new(); + let mut key_values: Vec = Vec::new(); + + for (token_index, token) in splitted_keys.enumerate() { + if token_index % 2 == 0 { + key_names.push(token.to_owned()); + } else { + key_values.push(token.trim().to_owned()); + } + } + + for (index, key_name) in key_names.iter().enumerate() { + let key_value = key_values[index].clone(); + cd_keys.insert(key_name.clone(), key_value); + } + } else if !raw_cd_keys_fix.is_empty() { + cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned()); + } + + cd_keys +} + +pub fn deserialize(content_id: u64, + content_string: &str, + os_filters: &Vec, + language_filters: &Vec, + resolution_filters: &Vec) + -> Result { + let content_raw: Value = serde_json::from_str(content_string)?; + debug!("found {:?}", &content_raw); + + let mut content: Content = serde_json::from_str(content_string)?; + content.id = content_id; + + let dlcs = &content_raw["dlcs"]; + let downloads = &content_raw["downloads"]; + + if content_raw.is_object() && + !content_raw + .as_object() + .unwrap() + .contains_key("forumLink") { + return Err(GogError::Error("No forumLink property")); + } + + let is_movie = match content_raw["forumLink"].as_str() { + Some(value) => value == "https://embed.gog.com/forum/movies", + None => false, + }; + + content.is_movie = is_movie; + + if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") { + let cd_keys_raw = content_raw["cdKey"].as_str().unwrap(); + content.cd_keys = deserialize_cd_keys(&content.title, cd_keys_raw); + } + + match dlcs.as_array() { + Some(value) => { + if value.len() > 0 { + debug!("processing dlc fields: {:?}", &value); + for dlc in value { + let dlc = deserialize(content_id, + serde_json::to_string(&dlc).unwrap().as_str(), + os_filters, + language_filters, + resolution_filters)?; + content.dlcs.push(dlc); + } + } + } + None => (), + } + + + debug!("processing installer fields: {:?}", &downloads); + for languages in downloads.as_array() { + for language in languages { + if !language.is_array() || language.as_array().unwrap().len() < 2 { + error!("Skipping a language for {}", content.title); + continue; + } + + let data_language = match language[0].as_str() { + Some(value) => value.to_lowercase(), + None => String::default(), + }; + + if data_language.is_empty() { + error!("Skipping a language for {}", content.title); + continue; + } + + if !is_movie && !language_filters.is_empty() && + !language_filters.contains(&data_language) { + info!("Skipping {} for {}", &data_language, content.title); + continue; + } + + for systems in language[1].as_object() { + for system in systems.keys() { + if is_movie && !os_filters.is_empty() && !os_filters.contains(system) { + info!("Skipping {} {} for {}", + &data_language, + system, + content.title); + continue; + } + + for real_downloads in systems.get(system) { + for real_download in real_downloads.as_array() { + for download in real_download { + if !download.is_object() || + !download.as_object().unwrap().contains_key("manualUrl") || + !download.as_object().unwrap().contains_key("name") { + error!("Skipping data for {}", content.title); + continue; + } + + let name: &str = download["name"].as_str().unwrap(); + + + if content.is_movie && !resolution_filters.is_empty() { + let mut found_resolution = false; + for resolution_filter in resolution_filters { + let filter = format!("({})", resolution_filter); + if name.ends_with(&filter) { + found_resolution = true; + break; + } + } + + if !found_resolution { + info!("Skipping {}: not a suitable resolution.", name); + continue; + } + } + + let data = Data { + manual_url: String::from(download["manualUrl"] + .as_str() + .unwrap()), + version: + String::from(download["version"].as_str().unwrap_or("")), + os: system.clone(), + language: data_language.clone(), + }; + + content.data.push(data); + } + } + } + } + } + } + } + + Ok(content) +}