add unmaintained note

break content deserializing and sync code into smaller pieces (fixes #9)

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.4 (2017-05-10)
+- sync dlcs (fixes #12)
+- break content deserializing and sync code into smaller pieces (fixes #9)
+
 ## 0.3.3 (2017-05-09)
 - normalize content directory names (fixes #11)
 

Cargo.lock

@@ -1,20 +1,3 @@
-[root]
-name = "gog-sync"
-version = "0.3.3"
-dependencies = [
- "chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "clap 2.24.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "curl 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "0.5.3"
@@ -128,6 +111,23 @@ dependencies = [
  "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "gog-sync"
+version = "0.3.4"
+dependencies = [
+ "chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "clap 2.24.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "curl 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "idna"
 version = "0.1.1"

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "gog-sync"
-version = "0.3.3"
+version = "0.3.4"
 authors = ["Sebastian Hugentobler <sebastian@vanwa.ch>"]
 description = "Synchronizes a GOG library with a local folder."
 documentation = "https://docs.rs/crate/gog-sync"
-repository = "https://gitlab.com/thallian/gog-sync"
+repository = "https://code.vanwa.ch/sebastian@vanwa.ch/gog-sync"
 readme = "README.md"
 keywords = ["gog", "sync", "cli"]
 categories = ["command-line-utilities"]

README.md

@@ -1,14 +1,21 @@
+# UNMAINTAINED AND BROKEN
+
+Used to work, does not anymore, maybe can be still of interest for someone
+
 # GOG-SYNC
+
 ## **Please note that this is alpha software, you should not trust it.**
 
 A small tool to synchronize the stuff in a [GOG](https://www.gog.com/) library
 with a local folder.
 
-It builds on the work of the [unofficial GOG API Documentation](https://gogapidocs.readthedocs.io/en/latest/).
+It builds on the work of the
+[unofficial GOG API Documentation](https://gogapidocs.readthedocs.io/en/latest/).
 
 This is the first time I am building something with rust, so beware :)
 
 # Installation
+
 Install from [crates.io](https://crates.io/crates/gog-sync).
 
 ```
@@ -16,8 +23,9 @@ cargo install gog-sync
 ```
 
 # Configuration
-The configuration file is in the config folder as described by the xdg specification
-with a prefix of `gog-sync`.
+
+The configuration file is in the config folder as described by the xdg
+specification with a prefix of `gog-sync`.
 
 For example on macOS or Linux
 
@@ -33,32 +41,38 @@ A bare configuration with default values before first use:
   "movieStorage": ".",
   "osFilters": [],
   "languageFilters": [],
+  "resolutionFilters": [],
   "skipMovies": false,
   "skipGames": false
 }
 ```
 
-- *gameStorage*: Where to save games
-- *movieStorage*: Where to save movies
-- *osFilters*: An array of operating systems. If it is not empty, game data is limited to the ones in the list.
-- *languageFilters*: An array of languages. If it is not empty, game data is limited to the ones in the list.
-- *resolutionFilters*: An array of resolutions. If it is not empty, movie data is limited to the ones in the list.
-- *skipMovies*: Whether to skip movie content
-- *skipGames*: Whether to skip game content
+-   _gameStorage_: Where to save games
+-   _movieStorage_: Where to save movies
+-   _osFilters_: An array of operating systems. If it is not empty, game data is
+    limited to the ones in the list.
+-   _languageFilters_: An array of languages. If it is not empty, game data is
+    limited to the ones in the list.
+-   _resolutionFilters_: An array of resolutions. If it is not empty, movie data
+    is limited to the ones in the list.
+-   _skipMovies_: Whether to skip movie content
+-   _skipGames_: Whether to skip game content
 
-Valid values for *osFilter*:
-- `linux`
-- `mac`
-- `windows`
+Valid values for _osFilter_:
+
+-   `linux`
+-   `mac`
+-   `windows`
 
 Check on [gog.com](https://www.gog.com/) which languages are available.
 
 An incomplete list of resolutions on gog:
-- `DVD`
-- `576p`
-- `720p`
-- `1080p`
-- `4k`
+
+-   `DVD`
+-   `576p`
+-   `720p`
+-   `1080p`
+-   `4k`
 
 # Usage
 
@@ -89,8 +103,8 @@ OPTIONS:
 gog-sync
 ```
 
-Normal invocation, uses the current working directory as storage if not configured
-otherwise.
+Normal invocation, uses the current working directory as storage if not
+configured otherwise.
 
 ---
 

src/gog.rs

@@ -15,17 +15,18 @@ use http::{Http, HttpError};
 use models;
 use models::content::Content;
 use models::contentinfo::ContentInfo;
-use models::datainfo::DataInfo;
-use models::extrainfo::ExtraInfo;
 use models::data::Data;
+use models::datainfo::DataInfo;
+use models::extra::Extra;
+use models::extrainfo::ExtraInfo;
 use models::token::Token;
 use serde_json;
 use serde_json::Value;
-use std::collections::HashMap;
 use std::collections::BTreeMap;
-use std::fmt;
+use std::collections::HashMap;
 use std::fs;
 use std::fs::File;
+use std::fmt;
 use std::io;
 use std::io::Write;
 use std::path::Path;
@@ -133,7 +134,7 @@ impl<'a> Gog<'a> {
     /// Syncs the contents of a gog account with a local folder.
     /// Uses a hash to figure out whether something has changed.
     pub fn sync(&mut self,
-                storage_path: &str,
+                storage_path_games: &str,
                 storage_path_movies: &str,
                 os_filters: &Vec<String>,
                 language_filters: &Vec<String>,
@@ -155,120 +156,200 @@ impl<'a> Gog<'a> {
                 }
             };
 
-            let content_root = if content.is_movie {
-                Path::new(storage_path_movies).join(&content.title)
-            } else {
-                Path::new(storage_path).join(&content.title)
-            };
-
-            let content_info_path = Path::new(&content_root).join("info.json");
-
-            let content_info_saved =
-                match ConfigFiles::load_from_path::<ContentInfo>(&content_info_path) {
-                    Ok(value) => value,
-                    Err(_) => ContentInfo::new(),
-                };
-
             if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) {
                 info!("filtering {}", content.title);
                 continue;
             }
 
-            let content_hash = models::get_hash(&content);
-
-            if content_info_saved.hash == content_hash {
-                info!("{} already up to date.", &content.title);
-                continue;
+            match self.sync_content(content, storage_path_games, storage_path_movies) {
+                Ok(_) => (),
+                Err(error) => {
+                    warn!("{:?}", error);
+                    continue;
+                }
             }
+        }
+        Ok(())
+    }
 
-            fs::create_dir_all(&content_root)?;
+    fn sync_content(&mut self,
+                    content: Content,
+                    storage_path_games: &str,
+                    storage_path_movies: &str)
+                    -> Result<(), GogError> {
 
-            let mut content_info = ContentInfo {
-                hash: content_hash,
-                id: content_id,
-                title: content.title.clone(),
-                cd_keys: content.cd_keys.clone(),
-                data: HashMap::new(),
-                extras: HashMap::new(),
+        let content_root = if content.is_movie {
+            Path::new(storage_path_movies).join(&content.title)
+        } else {
+            Path::new(storage_path_games).join(&content.title)
+        };
+
+        let content_info_path = Path::new(&content_root).join("info.json");
+        let content_info_saved =
+            match ConfigFiles::load_from_path::<ContentInfo>(&content_info_path) {
+                Ok(value) => value,
+                Err(_) => ContentInfo::new(),
             };
 
-            ConfigFiles::save_to_path(&content_info_path, &content_info)?;
+        let content_hash = models::get_hash(&content);
 
-            let key_root = content_root.join("keys");
+        if content_info_saved.hash == content_hash {
+            info!("{} already up to date.", &content.title);
+            return Ok(());
+        }
 
-            if content.cd_keys.len() > 0 {
-                fs::create_dir_all(&key_root)?;
-            }
+        fs::create_dir_all(&content_root)?;
 
-            for (key, value) in content.cd_keys {
-                let key_path = key_root.join(format!("{}.txt", key));
-                let mut key_file = File::create(&key_path)?;
-                key_file.write_all(value.as_bytes())?
-            }
+        let mut content_info = ContentInfo {
+            hash: content_hash,
+            id: content.id,
+            title: content.title.clone(),
+            cd_keys: content.cd_keys.clone(),
+            data: HashMap::new(),
+            extras: HashMap::new(),
+        };
 
-            for data in content.data {
-                let data_hash_saved = match content_info_saved.data.get(&content_id.to_string()) {
-                    Some(value) => value.hash,
-                    None => u64::min_value(),
-                };
+        ConfigFiles::save_to_path(&content_info_path, &content_info)?;
 
-                let data_hash = models::get_hash(&data);
+        self.save_keys(content.cd_keys, &content_root)?;
 
-                if data_hash_saved == data_hash {
-                    info!("{} already up to date.", &data.manual_url);
-                    continue;
-                }
+        for data in content.data {
+            self.save_data(content.title.as_str(),
+                           data,
+                           &content_root,
+                           &content_info_path,
+                           &content_info_saved,
+                           &mut content_info)?;
+        }
 
-                let data_root = Path::new(&content_root).join(&data.language);
-                fs::create_dir_all(&data_root)?;
+        let dlc_root = Path::new(&storage_path_games)
+            .join(&content.title)
+            .join("dlcs");
 
-                let data_uri = format!("https://embed.gog.com{}", data.manual_url);
+        for dlc in content.dlcs {
+            self.sync_content(dlc,
+                              dlc_root.to_string_lossy().as_ref(),
+                              storage_path_movies)?;
+        }
 
-                info!("downloading {} for {}...", &data.manual_url, &content.title);
-                let filename = self.api_request_download(data_uri.as_str(), &data_root)?;
+        for extra in content.extras {
+            self.save_extra(content.title.as_str(),
+                            extra,
+                            &content_root,
+                            &content_info_path,
+                            &content_info_saved,
+                            &mut content_info)?;
+        }
 
-                let data_info = DataInfo {
-                    hash: data_hash,
-                    filename: filename,
-                    language: data.language,
-                };
+        Ok(())
+    }
 
-                content_info
-                    .data
-                    .insert(data.manual_url.clone(), data_info);
-                ConfigFiles::save_to_path(&content_info_path, &content_info)?;
-            }
+    fn save_file(&mut self,
+                 content_title: &str,
+                 relative_uri: &str,
+                 data_root: &PathBuf)
+                 -> Result<String, GogError> {
+        fs::create_dir_all(&data_root)?;
+        let data_uri = format!("https://embed.gog.com{}", relative_uri);
 
-            for extra in content.extras {
-                let extra_hash_saved =
-                    match content_info_saved.extras.get(&content_id.to_string()) {
-                        Some(value) => value.hash,
-                        None => u64::min_value(),
-                    };
+        let filename = self.api_request_get_filename(&data_uri)?;
+        let file_path = Path::new(&data_root).join(&filename);
 
-                let extra_hash = models::get_hash(&extra);
+        if file_path.exists() {
+            info!("{} for {} already exists", &filename, content_title);
+            return Ok(filename.to_owned());
+        }
 
-                if extra_hash_saved == extra_hash {
-                    info!("{} already up to date.", &extra.manual_url);
-                    continue;
-                }
+        info!("downloading {} for {}...", &relative_uri, content_title);
+        self.api_request_download(data_uri.as_str(), &data_root)
+    }
 
-                let extra_uri = format!("https://embed.gog.com{}", extra.manual_url);
+    fn save_data(&mut self,
+                 content_title: &str,
+                 data: Data,
+                 content_root: &PathBuf,
+                 content_info_path: &PathBuf,
+                 content_info_saved: &ContentInfo,
+                 content_info: &mut ContentInfo)
+                 -> Result<(), GogError> {
+        let hash_saved = match content_info_saved.data.get(data.manual_url.as_str()) {
+            Some(value) => value.hash,
+            None => u64::min_value(),
+        };
 
-                info!("downloading {} for {}...", &extra.name, &content.title);
-                let filename = self.api_request_download(extra_uri.as_str(), &content_root)?;
+        let hash = models::get_hash(&data);
+        if hash_saved == hash {
+            info!("{} already up to date.", &data.manual_url);
+            return Ok(());
+        }
 
-                let extra_info = ExtraInfo {
-                    hash: extra_hash,
-                    filename: filename,
-                    name: extra.name,
-                };
+        let data_root = Path::new(&content_root).join(&data.language);
 
-                content_info
-                    .extras
-                    .insert(extra.manual_url.clone(), extra_info);
-                ConfigFiles::save_to_path(&content_info_path, &content_info)?;
-            }
+        let filename = self.save_file(content_title, &data.manual_url, &data_root)?;
+
+        let data_info = DataInfo {
+            hash: hash,
+            filename: filename,
+            language: data.language,
+        };
+
+        content_info
+            .data
+            .insert(data.manual_url.clone(), data_info);
+        ConfigFiles::save_to_path(&content_info_path, &content_info)?;
+
+        Ok(())
+    }
+
+    fn save_extra(&mut self,
+                  content_title: &str,
+                  extra: Extra,
+                  content_root: &PathBuf,
+                  content_info_path: &PathBuf,
+                  content_info_saved: &ContentInfo,
+                  content_info: &mut ContentInfo)
+                  -> Result<(), GogError> {
+        let hash_saved = match content_info_saved.data.get(extra.manual_url.as_str()) {
+            Some(value) => value.hash,
+            None => u64::min_value(),
+        };
+
+        let hash = models::get_hash(&extra);
+        if hash_saved == hash {
+            info!("{} already up to date.", &extra.manual_url);
+            return Ok(());
+        }
+
+        let filename = self.save_file(content_title, &extra.manual_url, &content_root)?;
+
+        let extra_info = ExtraInfo {
+            hash: hash,
+            filename: filename,
+            name: extra.name,
+        };
+
+        content_info
+            .extras
+            .insert(extra.manual_url.clone(), extra_info);
+        ConfigFiles::save_to_path(&content_info_path, &content_info)?;
+
+        Ok(())
+    }
+
+    fn save_keys(&mut self,
+                 cd_keys: BTreeMap<String, String>,
+                 content_root: &PathBuf)
+                 -> Result<(), GogError> {
+        let key_root = content_root.join("keys");
+
+        if cd_keys.len() > 0 {
+            fs::create_dir_all(&key_root)?;
+        }
+
+        for (key, value) in cd_keys {
+            let key_path = key_root.join(format!("{}.txt", key));
+            let mut key_file = File::create(&key_path)?;
+            key_file.write_all(value.as_bytes())?
         }
 
         Ok(())
@@ -349,45 +430,7 @@ impl<'a> Gog<'a> {
             content_ids.push(content_id_parsed);
         }
 
-        return Ok(content_ids);
-    }
-
-    fn parse_cd_keys(&self, content_title: &str, raw_cd_keys: &str) -> BTreeMap<String, String> {
-        let mut cd_keys = BTreeMap::new();
-        let mut raw_cd_keys_fix = raw_cd_keys.to_owned();
-
-        if raw_cd_keys_fix.contains("<span>") {
-            raw_cd_keys_fix = raw_cd_keys_fix
-                .replace("</span><span>", ":")
-                .replace("<span>", "")
-                .replace("</span>", "");
-        }
-
-        raw_cd_keys_fix = raw_cd_keys_fix.replace("<br>", ":").replace("::", ":");
-
-        if raw_cd_keys_fix.contains(":") {
-            let splitted_keys = raw_cd_keys_fix.split(":");
-
-            let mut key_names: Vec<String> = Vec::new();
-            let mut key_values: Vec<String> = Vec::new();
-
-            for (token_index, token) in splitted_keys.enumerate() {
-                if token_index % 2 == 0 {
-                    key_names.push(token.to_owned());
-                } else {
-                    key_values.push(token.trim().to_owned());
-                }
-            }
-
-            for (index, key_name) in key_names.iter().enumerate() {
-                let key_value = key_values[index].clone();
-                cd_keys.insert(key_name.clone(), key_value);
-            }
-        } else if !raw_cd_keys_fix.is_empty() {
-            cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned());
-        }
-
-        return cd_keys;
+        Ok(content_ids)
     }
 
     fn api_request_ensure_token(&mut self, response: &str) -> Result<bool, GogError> {
@@ -416,6 +459,15 @@ impl<'a> Gog<'a> {
         }
     }
 
+    fn api_request_get_filename(&mut self, uri: &str) -> Result<String, GogError> {
+        let response = self.http_client.get_filename(uri)?;
+        if self.api_request_ensure_token(&response)? {
+            Ok(self.http_client.get_filename(uri)?)
+        } else {
+            Ok(response)
+        }
+    }
+
     fn api_request_download(&mut self, uri: &str, path: &PathBuf) -> Result<String, GogError> {
         let response = self.http_client.download(uri, path)?;
         if self.api_request_ensure_token(&response)? {
@@ -436,118 +488,11 @@ impl<'a> Gog<'a> {
 
         let response = self.api_request_get(content_uri.as_str())?;
 
-        let content_raw: Value = serde_json::from_str(response.as_str())?;
-        debug!("found {:?}", &content_raw);
-
-        let mut content: Content = serde_json::from_str(&response)?;
-        content.id = content_id;
-
-        let downloads = &content_raw["downloads"];
-
-        if content_raw.is_object() &&
-           !content_raw
-                .as_object()
-                .unwrap()
-                .contains_key("forumLink") {
-            return Err(GogError::Error("No forumLink property"));
-        }
-
-        let is_movie = match content_raw["forumLink"].as_str() {
-            Some(value) => value == "https://embed.gog.com/forum/movies",
-            None => false,
-        };
-
-        content.is_movie = is_movie;
-
-        if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") {
-            let cd_keys_raw = content_raw["cdKey"].as_str().unwrap();
-            content.cd_keys = self.parse_cd_keys(&content.title, cd_keys_raw);
-        }
-
-        debug!("processing installer fields: {:?}", &downloads);
-        for languages in downloads.as_array() {
-            for language in languages {
-                if !language.is_array() || language.as_array().unwrap().len() < 2 {
-                    error!("Skipping a language for {}", content.title);
-                    continue;
-                }
-
-                let data_language = match language[0].as_str() {
-                    Some(value) => value.to_lowercase(),
-                    None => String::default(),
-                };
-
-                if data_language.is_empty() {
-                    error!("Skipping a language for {}", content.title);
-                    continue;
-                }
-
-                if !is_movie && !language_filters.is_empty() &&
-                   !language_filters.contains(&data_language) {
-                    info!("Skipping {} for {}", &data_language, content.title);
-                    continue;
-                }
-
-                for systems in language[1].as_object() {
-                    for system in systems.keys() {
-                        if is_movie && !os_filters.is_empty() && !os_filters.contains(system) {
-                            info!("Skipping {} {} for {}",
-                                  &data_language,
-                                  system,
-                                  content.title);
-                            continue;
-                        }
-
-                        for real_downloads in systems.get(system) {
-                            for real_download in real_downloads.as_array() {
-                                for download in real_download {
-                                    if !download.is_object() ||
-                                       !download.as_object().unwrap().contains_key("manualUrl") ||
-                                       !download.as_object().unwrap().contains_key("name") {
-                                        error!("Skipping data for {}", content.title);
-                                        continue;
-                                    }
-
-                                    let name: &str = download["name"].as_str().unwrap();
-
-
-                                    if content.is_movie && !resolution_filters.is_empty() {
-                                        let mut found_resolution = false;
-                                        for resolution_filter in resolution_filters {
-                                            let filter = format!("({})", resolution_filter);
-                                            if name.ends_with(&filter) {
-                                                found_resolution = true;
-                                                break;
-                                            }
-                                        }
-
-                                        if !found_resolution {
-                                            info!("Skipping {}: not a suitable resolution.", name);
-                                            continue;
-                                        }
-                                    }
-
-                                    let data = Data {
-                                        manual_url: String::from(download["manualUrl"]
-                                                                     .as_str()
-                                                                     .unwrap()),
-                                        version: String::from(download["version"]
-                                                                  .as_str()
-                                                                  .unwrap_or("")),
-                                        os: system.clone(),
-                                        language: data_language.clone(),
-                                    };
-
-                                    content.data.push(data);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        Ok(content)
+        models::content::deserialize(content_id,
+                                     response.as_str(),
+                                     os_filters,
+                                     language_filters,
+                                     resolution_filters)
     }
 
     fn games_uri(&self) -> String {

src/http.rs

@@ -96,10 +96,11 @@ impl Http {
         self.curl.url(uri)?;
         {
             let mut transfer = self.curl.transfer();
-            transfer.write_function(|new_data| {
-                    data.extend_from_slice(new_data);
-                    Ok(new_data.len())
-                })?;
+            transfer
+                .write_function(|new_data| {
+                                    data.extend_from_slice(new_data);
+                                    Ok(new_data.len())
+                                })?;
 
             transfer.perform()?;
         }
@@ -129,6 +130,46 @@ impl Http {
         }
     }
 
+    /// Find the filename for a download uri.
+    ///
+    /// Useful if the initial link gets redirected.
+    ///
+    /// The filename is taken from the last URI segment and returned in the result.
+    /// # Example
+    /// ```
+    /// use http::Http;
+    /// use std::path::Path;
+    ///
+    /// let mut http_client = Http::new();
+    ///
+    /// http_client.get_filename("https://example.com/sed");
+    /// ```
+    pub fn get_filename(&mut self, download_uri: &str) -> Result<String, HttpError> {
+        self.curl.url(download_uri)?;
+        self.curl.nobody(true)?;
+        self.curl.perform()?;
+        self.curl.nobody(false)?;
+
+        let download_url_string = match self.curl.effective_url()? {
+            Some(value) => value,
+            None => return Err(HttpError::Error("Can't get effective download url.")),
+        };
+
+        let download_url = Url::parse(download_url_string)?;
+
+        let download_url_segments = match download_url.path_segments() {
+            Some(value) => value,
+            None => return Err(HttpError::Error("Can't parse download segments.")),
+        };
+
+        let file_name = match download_url_segments.last() {
+            Some(value) => value,
+            None => return Err(HttpError::Error("No segments in download url.")),
+        };
+
+        Ok(file_name.to_owned())
+    }
+
     /// Download a file to the specified folder without creating the folder.
     ///
     /// The filename is taken from the last URI segment and returned in the result.
@@ -152,13 +193,14 @@ impl Http {
         self.curl.url(download_uri)?;
         {
             let mut transfer = self.curl.transfer();
-            transfer.write_function(|data| {
-                    match file_download.write(data) {
-                        Ok(_) => Ok(()),
-                        Err(error) => Err(HttpError::IOError(error)),
-                    }?;
-                    Ok(data.len())
-                })?;
+            transfer
+                .write_function(|data| {
+                                    match file_download.write(data) {
+                                        Ok(_) => Ok(()),
+                                        Err(error) => Err(HttpError::IOError(error)),
+                                    }?;
+                                    Ok(data.len())
+                                })?;
             transfer.perform()?;
         }
 

src/main.rs

@@ -29,7 +29,7 @@ fn main() {
     env_logger::init().unwrap();
 
     let matches = App::new("Gog Synchronizer")
-        .version("0.3.3")
+        .version("0.3.4")
         .author("Sebastian Hugentobler <sebastian@vanwa.ch>")
         .about("Synchronizes your gog library to a local folder.")
         .arg(Arg::with_name("game-storage")

src/models/content.rs

@@ -1,7 +1,10 @@
+use gog::GogError;
 use models::data::Data;
 use models::extra::Extra;
 use regex::Regex;
 use serde::{Deserialize, Deserializer};
+use serde_json;
+use serde_json::Value;
 use std::collections::BTreeMap;
 use std::fmt;
 
@@ -11,7 +14,7 @@ use std::fmt;
 pub struct Content {
     #[serde(skip_deserializing)]
     pub id: u64,
-    #[serde(deserialize_with = "normalize_title")]
+    #[serde(deserialize_with = "deserialize_title")]
     pub title: String,
     #[serde(skip_deserializing)]
     #[serde(rename(deserialize = "cdKey"))]
@@ -21,6 +24,7 @@ pub struct Content {
     #[serde(skip_deserializing)]
     pub data: Vec<Data>,
     pub extras: Vec<Extra>,
+    pub dlcs: Vec<Content>,
 }
 
 impl fmt::Display for Content {
@@ -29,7 +33,7 @@ impl fmt::Display for Content {
     }
 }
 
-fn normalize_title<D>(deserializer: D) -> Result<String, D::Error>
+fn deserialize_title<D>(deserializer: D) -> Result<String, D::Error>
     where D: Deserializer
 {
     let raw_title = String::deserialize(deserializer)?.replace(":", " - ");
@@ -47,3 +51,213 @@ fn normalize_title<D>(deserializer: D) -> Result<String, D::Error>
 
     Ok(title_whitespace)
 }
+
+/// Keys come in at least three different forms.
+/// It is possible to parse them all into the same structure, a map.
+///
+/// # Variants
+/// ## I
+/// Only one key and the value is the key itself.
+/// `1234-5678-1234-5678`
+///
+/// ## II
+/// Multiple keys split at single `<br>` tags with the key name and value itself
+/// split at `:<br>`.
+/// ```
+/// Neverwinter Nights:<br>	1234-5678-1234-5678	<br>Shadows of Undrentide:<br>	1234-5678-1234-5678	<br>Hordes of the Underdark:<br>	1234-5678-1234-5678
+/// ```
+///
+/// ## III
+/// Multiple keys split at `</span><span>` tags with the key name and value itself
+/// again split at `:<br>`.
+///
+/// ```
+/// <span>Neverwinter Nights 2:<br>1234-5678-1234-5678</span><span>Mask of the Betrayer:<br>1234-5678-1234-5678</span><span>Storm of Zehir:<br>1234-5678-1234-5678</span>
+/// ```
+///
+/// With this information the method works as follows:
+/// - replace every `</span><span>` with `:`
+/// - remove every `<span>` and `</span>`
+/// - replace every `<br>` with `:`
+/// - replace every `::` with `:`
+/// - if there is at least one `:`
+///     - split at `:`
+///     - every odd position in the resulting array is a key name, every even its corresponding value
+/// - else
+///     - the content name is the key name and the value is used as is
+///
+fn deserialize_cd_keys(content_title: &str, raw_cd_keys: &str) -> BTreeMap<String, String> {
+    let mut cd_keys = BTreeMap::new();
+    let mut raw_cd_keys_fix = raw_cd_keys.to_owned();
+
+    if raw_cd_keys_fix.contains("<span>") {
+        raw_cd_keys_fix = raw_cd_keys_fix
+            .replace("</span><span>", ":")
+            .replace("<span>", "")
+            .replace("</span>", "");
+    }
+
+    raw_cd_keys_fix = raw_cd_keys_fix.replace("<br>", ":").replace("::", ":");
+
+    if raw_cd_keys_fix.contains(":") {
+        let splitted_keys = raw_cd_keys_fix.split(":");
+
+        let mut key_names: Vec<String> = Vec::new();
+        let mut key_values: Vec<String> = Vec::new();
+
+        for (token_index, token) in splitted_keys.enumerate() {
+            if token_index % 2 == 0 {
+                key_names.push(token.to_owned());
+            } else {
+                key_values.push(token.trim().to_owned());
+            }
+        }
+
+        for (index, key_name) in key_names.iter().enumerate() {
+            let key_value = key_values[index].clone();
+            cd_keys.insert(key_name.clone(), key_value);
+        }
+    } else if !raw_cd_keys_fix.is_empty() {
+        cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned());
+    }
+
+    cd_keys
+}
+
+pub fn deserialize(content_id: u64,
+                   content_string: &str,
+                   os_filters: &Vec<String>,
+                   language_filters: &Vec<String>,
+                   resolution_filters: &Vec<String>)
+                   -> Result<Content, GogError> {
+    let content_raw: Value = serde_json::from_str(content_string)?;
+    debug!("found {:?}", &content_raw);
+
+    let mut content: Content = serde_json::from_str(content_string)?;
+    content.id = content_id;
+
+    let dlcs = &content_raw["dlcs"];
+    let downloads = &content_raw["downloads"];
+
+    if content_raw.is_object() &&
+       !content_raw
+            .as_object()
+            .unwrap()
+            .contains_key("forumLink") {
+        return Err(GogError::Error("No forumLink property"));
+    }
+
+    let is_movie = match content_raw["forumLink"].as_str() {
+        Some(value) => value == "https://embed.gog.com/forum/movies",
+        None => false,
+    };
+
+    content.is_movie = is_movie;
+
+    if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") {
+        let cd_keys_raw = content_raw["cdKey"].as_str().unwrap();
+        content.cd_keys = deserialize_cd_keys(&content.title, cd_keys_raw);
+    }
+
+    match dlcs.as_array() {
+        Some(value) => {
+            if value.len() > 0 {
+                debug!("processing dlc fields: {:?}", &value);
+                for dlc in value {
+                    let dlc = deserialize(content_id,
+                                          serde_json::to_string(&dlc).unwrap().as_str(),
+                                          os_filters,
+                                          language_filters,
+                                          resolution_filters)?;
+                    content.dlcs.push(dlc);
+                }
+            }
+        }
+        None => (),
+    }
+
+
+    debug!("processing installer fields: {:?}", &downloads);
+    for languages in downloads.as_array() {
+        for language in languages {
+            if !language.is_array() || language.as_array().unwrap().len() < 2 {
+                error!("Skipping a language for {}", content.title);
+                continue;
+            }
+
+            let data_language = match language[0].as_str() {
+                Some(value) => value.to_lowercase(),
+                None => String::default(),
+            };
+
+            if data_language.is_empty() {
+                error!("Skipping a language for {}", content.title);
+                continue;
+            }
+
+            if !is_movie && !language_filters.is_empty() &&
+               !language_filters.contains(&data_language) {
+                info!("Skipping {} for {}", &data_language, content.title);
+                continue;
+            }
+
+            for systems in language[1].as_object() {
+                for system in systems.keys() {
+                    if is_movie && !os_filters.is_empty() && !os_filters.contains(system) {
+                        info!("Skipping {} {} for {}",
+                              &data_language,
+                              system,
+                              content.title);
+                        continue;
+                    }
+
+                    for real_downloads in systems.get(system) {
+                        for real_download in real_downloads.as_array() {
+                            for download in real_download {
+                                if !download.is_object() ||
+                                   !download.as_object().unwrap().contains_key("manualUrl") ||
+                                   !download.as_object().unwrap().contains_key("name") {
+                                    error!("Skipping data for {}", content.title);
+                                    continue;
+                                }
+
+                                let name: &str = download["name"].as_str().unwrap();
+
+
+                                if content.is_movie && !resolution_filters.is_empty() {
+                                    let mut found_resolution = false;
+                                    for resolution_filter in resolution_filters {
+                                        let filter = format!("({})", resolution_filter);
+                                        if name.ends_with(&filter) {
+                                            found_resolution = true;
+                                            break;
+                                        }
+                                    }
+
+                                    if !found_resolution {
+                                        info!("Skipping {}: not a suitable resolution.", name);
+                                        continue;
+                                    }
+                                }
+
+                                let data = Data {
+                                    manual_url: String::from(download["manualUrl"]
+                                                                 .as_str()
+                                                                 .unwrap()),
+                                    version:
+                                        String::from(download["version"].as_str().unwrap_or("")),
+                                    os: system.clone(),
+                                    language: data_language.clone(),
+                                };
+
+                                content.data.push(data);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(content)
+}