From 88827749db299c42482ec8edabbf29890d9237c2 Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Tue, 21 Mar 2017 22:28:03 +0100 Subject: [PATCH 1/7] initial movie support --- Cargo.lock | 2 +- src/configfiles.rs | 10 ++++++++++ src/gog.rs | 47 ++++++++++++++++++++++++++++++++++++++++++---- src/http.rs | 13 +++++++++++++ src/main.rs | 17 ++++++++++++++++- src/models.rs | 4 ++++ 6 files changed, 87 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5787265..8af871c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ [root] name = "gog-sync" -version = "0.2.0" +version = "0.2.1" dependencies = [ "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/src/configfiles.rs b/src/configfiles.rs index 6eb1ba9..2915a32 100644 --- a/src/configfiles.rs +++ b/src/configfiles.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_json; +use std::fmt; use std::fs::File; use std; use std::io::{Read, Write}; @@ -20,6 +21,15 @@ pub enum ConfigError { SerdeJsonError(serde_json::Error), } +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + ConfigError::IOError(ref err) => fmt::Display::fmt(err, f), + ConfigError::SerdeJsonError(ref err) => fmt::Display::fmt(err, f), + } + } +} + impl From for ConfigError { fn from(e: std::io::Error) -> Self { ConfigError::IOError(e) diff --git a/src/gog.rs b/src/gog.rs index a9bead7..edbeab0 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -17,6 +17,7 @@ use models::{Token, Game, Installer, Config}; use serde_json; use serde_json::Value; use std::collections::HashMap; +use std::fmt; use std::fs; use std::fs::File; use std::io; @@ -33,6 +34,18 @@ pub enum GogError { IOError(io::Error), } +impl fmt::Display for GogError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + GogError::Error(ref err) => fmt::Display::fmt(err, f), + GogError::ConfigError(ref err) => fmt::Display::fmt(err, f), + GogError::HttpError(ref err) => fmt::Display::fmt(err, f), + GogError::SerdeError(ref err) => fmt::Display::fmt(err, f), + GogError::IOError(ref err) => fmt::Display::fmt(err, f), + } + } +} + impl From for GogError { fn from(e: ConfigError) -> Self { GogError::ConfigError(e) @@ -109,6 +122,7 @@ impl<'a> Gog<'a> { /// Uses a hash to figure out whether something has changed. pub fn sync(&mut self, storage_path: &str, + storage_path_movies: &str, os_filters: &Vec, language_filters: &Vec) -> Result<(), GogError> { @@ -120,6 +134,7 @@ impl<'a> Gog<'a> { Config { storage: String::from(storage_path), + movie_storage: String::from(storage_path_movies), games: HashMap::new(), installers: HashMap::new(), extras: HashMap::new(), @@ -137,7 +152,14 @@ impl<'a> Gog<'a> { None => u64::min_value(), }; - let game = self.get_game(game_id, &os_filters, &language_filters)?; + let game = match self.get_game(game_id, &os_filters, &language_filters) { + Ok(value) => value, + Err(error) => { + error!("{}", error); + break; + } + }; + let game_hash = models::get_hash(&game); if game_hash_saved == game_hash { @@ -145,7 +167,12 @@ impl<'a> Gog<'a> { continue; } - let game_root = Path::new(storage_path).join(&game.title); + let game_root = if game.is_movie { + Path::new(storage_path_movies).join(&game.title) + } else { + Path::new(storage_path).join(&game.title) + }; + fs::create_dir_all(&game_root)?; if !game.cd_key.is_empty() { @@ -306,6 +333,17 @@ impl<'a> Gog<'a> { let game_raw: Value = serde_json::from_str(response.as_str())?; let downloads = &game_raw["downloads"]; + if game_raw.is_object() && !game_raw.as_object().unwrap().contains_key("forumLink") { + return Err(GogError::Error("No forumLink property")); + } + + let is_movie = match game_raw["forumLink"].as_str() { + Some(value) => value == "https://embed.gog.com/forum/movies", + None => false, + }; + + game.is_movie = is_movie; + for languages in downloads.as_array() { for language in languages { if !language.is_array() || language.as_array().unwrap().len() < 2 { @@ -323,14 +361,15 @@ impl<'a> Gog<'a> { continue; } - if !language_filters.is_empty() && !language_filters.contains(&installer_language) { + if !is_movie && !language_filters.is_empty() && + !language_filters.contains(&installer_language) { info!("Skipping {} for {}", &installer_language, game.title); continue; } for systems in language[1].as_object() { for system in systems.keys() { - if !os_filters.is_empty() && !os_filters.contains(system) { + if is_movie && !os_filters.is_empty() && !os_filters.contains(system) { info!("Skipping {} {} for {}", &installer_language, system, diff --git a/src/http.rs b/src/http.rs index 5fdd8c2..a43b262 100644 --- a/src/http.rs +++ b/src/http.rs @@ -3,6 +3,7 @@ use curl; use curl::easy::{Easy, List, WriteError}; +use std::fmt; use std::fs; use std::fs::File; use std::io; @@ -24,6 +25,18 @@ pub enum HttpError { UrlParseError(url::ParseError), } +impl fmt::Display for HttpError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + HttpError::Error(ref err) => fmt::Display::fmt(err, f), + HttpError::CurlError(ref err) => fmt::Display::fmt(err, f), + HttpError::Utf8Error(ref err) => fmt::Display::fmt(err, f), + HttpError::IOError(ref err) => fmt::Display::fmt(err, f), + HttpError::UrlParseError(ref err) => fmt::Display::fmt(err, f), + } + } +} + impl From for HttpError { fn from(e: curl::Error) -> Self { HttpError::CurlError(e) diff --git a/src/main.rs b/src/main.rs index 442079a..81c91e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,6 +52,12 @@ fn main() { .value_name("FOLDER") .help("Sets the download folder (defaults to the working directory).") .takes_value(true)) + .arg(Arg::with_name("movie-storage") + .short("m") + .long("movie-storage") + .value_name("FOLDER") + .help("Sets the download folder for movies (defaults to the game directory).") + .takes_value(true)) .arg(Arg::with_name("os") .short("o") .long("os") @@ -78,6 +84,11 @@ fn main() { None => config.storage.as_str(), }; + let download_folder_movies: &str = match matches.value_of("movieStorage") { + Some(value) => value, + None => config.movie_storage.as_str(), + }; + let os_filters: Vec = match matches.value_of("os") { Some(value) => value.split(',').map(String::from).collect(), None => config.os_filters, @@ -91,5 +102,9 @@ fn main() { let mut http_client = Http::new(); let mut gog = Gog::new(&mut http_client); gog.login().unwrap(); - gog.sync(download_folder, &os_filters, &language_filters).unwrap(); + gog.sync(download_folder, + download_folder_movies, + &os_filters, + &language_filters) + .unwrap(); } diff --git a/src/models.rs b/src/models.rs index 9b2ab09..e577f58 100644 --- a/src/models.rs +++ b/src/models.rs @@ -33,6 +33,7 @@ fn timestamp() -> i64 { #[serde(rename_all = "camelCase")] pub struct Config { pub storage: String, + pub movie_storage: String, #[serde(default = "default_map")] pub games: HashMap, #[serde(default = "default_map")] @@ -57,6 +58,7 @@ impl Config { pub fn new() -> Config { Config { storage: String::from("."), + movie_storage: String::from("."), games: HashMap::new(), installers: HashMap::new(), extras: HashMap::new(), @@ -73,6 +75,8 @@ pub struct Game { pub title: String, pub cd_key: String, #[serde(skip_deserializing)] + pub is_movie: bool, + #[serde(skip_deserializing)] pub installers: Vec, pub extras: Vec, } From d538536af8fa63d27a6180403c2e4618a3043118 Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Wed, 22 Mar 2017 09:40:48 +0100 Subject: [PATCH 2/7] add default values to config struct --- src/models.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/models.rs b/src/models.rs index e577f58..6f41c92 100644 --- a/src/models.rs +++ b/src/models.rs @@ -32,7 +32,9 @@ fn timestamp() -> i64 { #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Config { + #[serde(default)] pub storage: String, + #[serde(default)] pub movie_storage: String, #[serde(default = "default_map")] pub games: HashMap, From 569491a9ab9ce2cfb6a8d6f490b71f854ff15e00 Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Wed, 22 Mar 2017 09:56:25 +0100 Subject: [PATCH 3/7] update changelog --- CHANGELOG.md | 3 +++ Cargo.toml | 2 +- src/main.rs | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5850b32..5dbcf33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.2.2 (2017-03-22) +- default value for storage config (fixes #6) + ## 0.2.1 (2017-03-21) - add instructions for getting the code if there is no token diff --git a/Cargo.toml b/Cargo.toml index c3c9ae2..5d33ed1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gog-sync" -version = "0.2.1" +version = "0.2.2" authors = ["Sebastian Hugentobler "] description = "Synchronizes a GOG library with a local folder." documentation = "https://docs.rs/crate/gog-sync" diff --git a/src/main.rs b/src/main.rs index 442079a..1f6606f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -43,7 +43,7 @@ fn main() { env_logger::init().unwrap(); let matches = App::new("Gog Synchronizer") - .version("0.2.1") + .version("0.2.2") .author("Sebastian Hugentobler ") .about("Synchronizes your gog library to a local folder.") .arg(Arg::with_name("storage") From bcb95f3c2895e235b8a70d2b95e33a03301ab970 Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Wed, 22 Mar 2017 14:11:57 +0100 Subject: [PATCH 4/7] take the whole http response and not only the last part --- Cargo.lock | 2 +- src/gog.rs | 6 +++++- src/http.rs | 14 ++++++++------ src/main.rs | 4 +++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8af871c..0a50d85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ [root] name = "gog-sync" -version = "0.2.1" +version = "0.2.2" dependencies = [ "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/src/gog.rs b/src/gog.rs index edbeab0..06d520e 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -325,12 +325,15 @@ impl<'a> Gog<'a> { language_filters: &Vec) -> Result { let game_uri = self.game_uri(game_id); + debug!("looking for information at {}...", &game_uri); let response = self.http_client.get(game_uri.as_str())?; + let game_raw: Value = serde_json::from_str(response.as_str())?; + debug!("found {:?}", &game_raw); + let mut game: Game = serde_json::from_str(&response)?; - let game_raw: Value = serde_json::from_str(response.as_str())?; let downloads = &game_raw["downloads"]; if game_raw.is_object() && !game_raw.as_object().unwrap().contains_key("forumLink") { @@ -344,6 +347,7 @@ impl<'a> Gog<'a> { game.is_movie = is_movie; + debug!("processing installer fields: {:?}", &downloads); for languages in downloads.as_array() { for language in languages { if !language.is_array() || language.as_array().unwrap().len() < 2 { diff --git a/src/http.rs b/src/http.rs index a43b262..a4b40ca 100644 --- a/src/http.rs +++ b/src/http.rs @@ -92,20 +92,22 @@ impl Http { /// http_client.get("https://discworld.com/"); /// ``` pub fn get(&mut self, uri: &str) -> Result { - let mut response_body = String::new(); - + let mut data = Vec::new(); self.curl.url(uri)?; { let mut transfer = self.curl.transfer(); - transfer.write_function(|data| { - response_body = String::from(str::from_utf8(data).unwrap()); - Ok(data.len()) + transfer.write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) })?; transfer.perform()?; } - Ok(response_body) + match str::from_utf8(data.as_slice()) { + Ok(value) => Ok(value.to_owned()), + Err(error) => Err(HttpError::Utf8Error(error)), + } } /// Add a header to all future requests. diff --git a/src/main.rs b/src/main.rs index 7ba17c0..9ae3ee8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,9 +11,11 @@ //! OPTIONS: //! -l, --language Only sync files for this comma seperated list of languages. //! -o, --os Only sync files for this comma seperated list of operating systems. -//! Valid values are 'linux', 'mac' and 'windows'. +//! Valid values are linux, mac and windows. //! -s, --storage Sets the download folder (defaults to the working directory). //! ``` +//! +//! An incomplete list of languages on gog: `english, český, deutsch, español, français, magyar, polski, русский, 中文` extern crate chrono; extern crate clap; From c1779265af924906f7df7f5fe8fb6e387e56ebc2 Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Wed, 22 Mar 2017 14:40:54 +0100 Subject: [PATCH 5/7] ability to skip game or movie content --- src/gog.rs | 11 ++++++++++- src/main.rs | 30 +++++++++++++++++++++++++++--- src/models.rs | 6 ++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/gog.rs b/src/gog.rs index 06d520e..33d4b71 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -124,7 +124,9 @@ impl<'a> Gog<'a> { storage_path: &str, storage_path_movies: &str, os_filters: &Vec, - language_filters: &Vec) + language_filters: &Vec, + skip_movies: bool, + skip_games: bool) -> Result<(), GogError> { let configfiles = ConfigFiles::new(); let mut config: Config = match configfiles.load("config.json") { @@ -140,6 +142,8 @@ impl<'a> Gog<'a> { extras: HashMap::new(), os_filters: os_filters.clone(), language_filters: language_filters.clone(), + skip_movies: skip_movies, + skip_games: skip_games, } } }; @@ -160,6 +164,11 @@ impl<'a> Gog<'a> { } }; + if (game.is_movie && skip_movies) || (!game.is_movie && skip_games) { + info!("filtering {}", game.title); + continue; + } + let game_hash = models::get_hash(&game); if game_hash_saved == game_hash { diff --git a/src/main.rs b/src/main.rs index 9ae3ee8..fc9ca41 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,7 @@ //! -s, --storage Sets the download folder (defaults to the working directory). //! ``` //! -//! An incomplete list of languages on gog: `english, český, deutsch, español, français, magyar, polski, русский, 中文` +//! An incomplete list of languages on gog: `english, český, deutsch, español, français, italiano, magyar, polski, русский, 中文` extern crate chrono; extern crate clap; @@ -73,6 +73,16 @@ fn main() { .value_name("FILTER") .help("Only sync files for this comma seperated list of languages.") .takes_value(true)) + .arg(Arg::with_name("skip-movies") + .short("f") + .long("skip-movies") + .help("Skip movie content.") + .takes_value(false)) + .arg(Arg::with_name("skip-games") + .short("g") + .long("skip-games") + .help("Skip game content.") + .takes_value(false)) .get_matches(); let configfiles = ConfigFiles::new(); @@ -88,7 +98,7 @@ fn main() { let download_folder_movies: &str = match matches.value_of("movieStorage") { Some(value) => value, - None => config.movie_storage.as_str(), + None => download_folder, }; let os_filters: Vec = match matches.value_of("os") { @@ -101,12 +111,26 @@ fn main() { None => config.language_filters, }; + let skip_movies = if matches.is_present("skip-movies") { + true + } else { + config.skip_movies + }; + + let skip_games = if matches.is_present("skip-games") { + true + } else { + config.skip_games + }; + let mut http_client = Http::new(); let mut gog = Gog::new(&mut http_client); gog.login().unwrap(); gog.sync(download_folder, download_folder_movies, &os_filters, - &language_filters) + &language_filters, + skip_movies, + skip_games) .unwrap(); } diff --git a/src/models.rs b/src/models.rs index 6f41c92..2e3e43a 100644 --- a/src/models.rs +++ b/src/models.rs @@ -46,6 +46,10 @@ pub struct Config { pub os_filters: Vec, #[serde(default = "default_list")] pub language_filters: Vec, + #[serde(default)] + pub skip_movies: bool, + #[serde(default)] + pub skip_games: bool, } fn default_map() -> HashMap { @@ -66,6 +70,8 @@ impl Config { extras: HashMap::new(), os_filters: Vec::new(), language_filters: Vec::new(), + skip_movies: false, + skip_games: false, } } } From 0ec9a41bff2537a00e57c11d437c4d7530b1ff89 Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Wed, 22 Mar 2017 15:39:46 +0100 Subject: [PATCH 6/7] refactor naming to be more generic --- .editorconfig | 3 + CHANGELOG.md | 3 + README.md | 62 +++++++++++++++++-- src/gog.rs | 164 +++++++++++++++++++++++++------------------------- src/main.rs | 14 ++--- src/models.rs | 22 +++---- 6 files changed, 163 insertions(+), 105 deletions(-) diff --git a/.editorconfig b/.editorconfig index d741e40..fa9ce7c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,3 +7,6 @@ trim_trailing_whitespace = true insert_final_newline = true indent_style = space indent_size = 4 + +[*.md] +trim_trailing_whitespace = true diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dbcf33..606fad1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 0.2.2 (2017-03-22) +- ability to save movies seperately - default value for storage config (fixes #6) +- ability to skip movie and game content +- fix a bug with truncating an http response ## 0.2.1 (2017-03-21) - add instructions for getting the code if there is no token diff --git a/README.md b/README.md index 1deb100..b674676 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # GOG-SYNC +## **Please note that this is alpha software, you should not trust it.** A small tool to synchronize the stuff in a [GOG](https://www.gog.com/) library with a local folder. @@ -7,8 +8,6 @@ It builds on the work of the [unofficial GOG API Documentation](https://gogapido This is the first time I am building something with rust, so beware :) -**Please note that this is alpha software, you should not trust it.** - # Installation Install from [crates.io](https://crates.io). @@ -26,13 +25,68 @@ For example on macOS or Linux ~/.config/gog-sync/config.json ``` -It is in Json format and the only relevant key is `storage`. The rest is information -about content hashes. +A bare configuration with default values before first use: +``` +{ + "gameStorage": ".", + "movieStorage": ".", + "content": {}, + "data": {}, + "extras": {}, + "osFilters": [], + "languageFilters": [], + "skipMovies": false, + "skipGames": false +} +``` + +- *gameStorage*: Where to save games +- *movieStorage*: Where to save movies +- *content*: A map, content id => hash +- *data*: A map, data url => hash +- *extras*: A map, extra url => hash +- *osFilters*: An array of operating systems. If it is not empty, game data is limited to the ones in the list. +Valid values are `linux`, `mac` and `windows` +- *languageFilters*: An array of languages. If it is not empty, game data is limited to the ones in the list. +- *skipMovies*: Whether to skip movie content +- *skipGames*: Whether to skip game content + +An incomplete list of languages on gog: +- `english` +- `český` +- `deutsch` +- `español` +- `français` +- `italiano` +- `magyar` +- `polski` +- `русский` +- `中文` + +You should have no need of changing `content`, `data` or `extras`, as these are +used to determine whether specific content is up to date. # Usage If you want to see the information log while running set `RUST_LOG=info`. +``` +LAGS] [OPTIONS] + +FLAGS: + -h, --help Prints help information + -g, --skip-games Skip game content. + -f, --skip-movies Skip movie content. + -V, --version Prints version information + +OPTIONS: + -s, --game-storage Sets the download folder (defaults to the working directory). + -l, --language Only sync files for this comma seperated list of languages. + -m, --movie-storage Sets the download folder for movies (defaults to the working directory). + -o, --os Only sync files for this comma seperated list of operating systems. + Valid values are 'linux', 'mac' and 'windows'. +``` + --- ``` diff --git a/src/gog.rs b/src/gog.rs index 33d4b71..6190fa7 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -13,7 +13,7 @@ use configfiles::{ConfigFiles, ConfigError}; use http::{Http, HttpError}; use models; -use models::{Token, Game, Installer, Config}; +use models::{Token, Content, Data, Config}; use serde_json; use serde_json::Value; use std::collections::HashMap; @@ -135,10 +135,10 @@ impl<'a> Gog<'a> { error!("Configuration error, generating new one..."); Config { - storage: String::from(storage_path), + game_storage: String::from(storage_path), movie_storage: String::from(storage_path_movies), - games: HashMap::new(), - installers: HashMap::new(), + content: HashMap::new(), + data: HashMap::new(), extras: HashMap::new(), os_filters: os_filters.clone(), language_filters: language_filters.clone(), @@ -148,75 +148,73 @@ impl<'a> Gog<'a> { } }; - let game_ids = self.get_game_ids()?; + let content_ids = self.get_content_ids()?; - for game_id in game_ids { - let game_hash_saved = match config.games.get(&game_id.to_string()) { + for content_id in content_ids { + let content_hash_saved = match config.content.get(&content_id.to_string()) { Some(value) => value.clone(), None => u64::min_value(), }; - let game = match self.get_game(game_id, &os_filters, &language_filters) { + let content = match self.get_content(content_id, &os_filters, &language_filters) { Ok(value) => value, Err(error) => { - error!("{}", error); - break; + error!("{}: {}", &content_id, error); + continue; } }; - if (game.is_movie && skip_movies) || (!game.is_movie && skip_games) { - info!("filtering {}", game.title); + if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { + info!("filtering {}", content.title); continue; } - let game_hash = models::get_hash(&game); + let content_hash = models::get_hash(&content); - if game_hash_saved == game_hash { - info!("{} already up to date.", &game.title); + if content_hash_saved == content_hash { + info!("{} already up to date.", &content.title); continue; } - let game_root = if game.is_movie { - Path::new(storage_path_movies).join(&game.title) + let content_root = if content.is_movie { + Path::new(storage_path_movies).join(&content.title) } else { - Path::new(storage_path).join(&game.title) + Path::new(storage_path).join(&content.title) }; - fs::create_dir_all(&game_root)?; + fs::create_dir_all(&content_root)?; - if !game.cd_key.is_empty() { - let key_path = Path::new(game_root.as_os_str()).join("key.txt"); + if !content.cd_key.is_empty() { + let key_path = Path::new(content_root.as_os_str()).join("key.txt"); let mut key_file = File::create(&key_path)?; - key_file.write_all(game.cd_key.as_bytes())? + key_file.write_all(content.cd_key.as_bytes())? } - for installer in game.installers { - let installer_hash_saved = match config.installers.get(&installer.manual_url) { + for data in content.data { + let data_hash_saved = match config.data.get(&data.manual_url) { Some(value) => value.clone(), None => u64::min_value(), }; - let installer_hash = models::get_hash(&installer); + let data_hash = models::get_hash(&data); - if installer_hash_saved == installer_hash { - info!("{} already up to date.", &installer.manual_url); + if data_hash_saved == data_hash { + info!("{} already up to date.", &data.manual_url); continue; } - let installer_root = Path::new(&game_root).join(&installer.language); - fs::create_dir_all(&installer_root)?; + let data_root = Path::new(&content_root).join(&data.language); + fs::create_dir_all(&data_root)?; - let installer_uri = format!("https://embed.gog.com{}", installer.manual_url); + let data_uri = format!("https://embed.gog.com{}", data.manual_url); - info!("downloading {} for {}...", - &installer.manual_url, - &game.title); - self.http_client.download(installer_uri.as_str(), &installer_root)?; + info!("downloading {} for {}...", &data.manual_url, &content.title); + self.http_client.download(data_uri.as_str(), &data_root)?; - config.installers.insert(installer.manual_url, installer_hash); + config.data.insert(data.manual_url, data_hash); configfiles.save("config.json", &config)?; } - for extra in game.extras { + for extra in content.extras { let extra_hash_saved = match config.extras.get(&extra.manual_url) { Some(value) => value.clone(), None => u64::min_value(), @@ -231,14 +229,14 @@ impl<'a> Gog<'a> { let extra_uri = format!("https://embed.gog.com{}", extra.manual_url); - info!("downloading {} for {}...", &extra.name, &game.title); - self.http_client.download(extra_uri.as_str(), &game_root)?; + info!("downloading {} for {}...", &extra.name, &content.title); + self.http_client.download(extra_uri.as_str(), &content_root)?; config.extras.insert(extra.manual_url, extra_hash); configfiles.save("config.json", &config)?; } - config.games.insert(game_id.to_string(), game_hash); + config.content.insert(content_id.to_string(), content_hash); configfiles.save("config.json", &config)?; } @@ -297,86 +295,86 @@ impl<'a> Gog<'a> { } } - fn get_game_ids(&mut self) -> Result, GogError> { + fn get_content_ids(&mut self) -> Result, GogError> { let response = self.http_client.get(self.games_uri.as_str())?; - let game_ids_raw: Value = serde_json::from_str(response.as_str())?; - let game_ids_serde = &game_ids_raw["owned"]; + let content_ids_raw: Value = serde_json::from_str(response.as_str())?; + let content_ids_serde = &content_ids_raw["owned"]; - let mut game_ids: Vec = Vec::new(); + let mut content_ids: Vec = Vec::new(); - if !game_ids_serde.is_array() { - return Err(GogError::Error("Error parsing game ids.")); + if !content_ids_serde.is_array() { + return Err(GogError::Error("Error parsing content ids.")); } - for game_id in game_ids_serde.as_array().unwrap() { - let game_id_parsed = game_id.as_u64().unwrap_or(0); + for content_id in content_ids_serde.as_array().unwrap() { + let content_id_parsed = content_id.as_u64().unwrap_or(0); - if game_id_parsed == 0 { - error!("Cant parse game id {}", game_id); + if content_id_parsed == 0 { + error!("Cant parse content id {}", content_id); continue; } // The numbers in this list are excluded because they refer to // favourites, promotions and such. - if [1, 2, 3, 4, 5].contains(&game_id_parsed) { + if [1, 2, 3, 4, 5].contains(&content_id_parsed) { continue; } - game_ids.push(game_id_parsed); + content_ids.push(content_id_parsed); } - return Ok(game_ids); + return Ok(content_ids); } - fn get_game(&mut self, - game_id: u64, - os_filters: &Vec, - language_filters: &Vec) - -> Result { - let game_uri = self.game_uri(game_id); - debug!("looking for information at {}...", &game_uri); + fn get_content(&mut self, + content_id: u64, + os_filters: &Vec, + language_filters: &Vec) + -> Result { + let content_uri = self.content_uri(content_id); + debug!("looking for information at {}...", &content_uri); - let response = self.http_client.get(game_uri.as_str())?; + let response = self.http_client.get(content_uri.as_str())?; - let game_raw: Value = serde_json::from_str(response.as_str())?; - debug!("found {:?}", &game_raw); + let content_raw: Value = serde_json::from_str(response.as_str())?; + debug!("found {:?}", &content_raw); - let mut game: Game = serde_json::from_str(&response)?; + let mut content: Content = serde_json::from_str(&response)?; - let downloads = &game_raw["downloads"]; + let downloads = &content_raw["downloads"]; - if game_raw.is_object() && !game_raw.as_object().unwrap().contains_key("forumLink") { + if content_raw.is_object() && !content_raw.as_object().unwrap().contains_key("forumLink") { return Err(GogError::Error("No forumLink property")); } - let is_movie = match game_raw["forumLink"].as_str() { + let is_movie = match content_raw["forumLink"].as_str() { Some(value) => value == "https://embed.gog.com/forum/movies", None => false, }; - game.is_movie = is_movie; + content.is_movie = is_movie; debug!("processing installer fields: {:?}", &downloads); for languages in downloads.as_array() { for language in languages { if !language.is_array() || language.as_array().unwrap().len() < 2 { - error!("Skipping a language for {}", game.title); + error!("Skipping a language for {}", content.title); continue; } - let installer_language = match language[0].as_str() { + let data_language = match language[0].as_str() { Some(value) => value.to_lowercase(), None => String::default(), }; - if installer_language.is_empty() { - error!("Skipping a language for {}", game.title); + if data_language.is_empty() { + error!("Skipping a language for {}", content.title); continue; } if !is_movie && !language_filters.is_empty() && - !language_filters.contains(&installer_language) { - info!("Skipping {} for {}", &installer_language, game.title); + !language_filters.contains(&data_language) { + info!("Skipping {} for {}", &data_language, content.title); continue; } @@ -384,9 +382,9 @@ impl<'a> Gog<'a> { for system in systems.keys() { if is_movie && !os_filters.is_empty() && !os_filters.contains(system) { info!("Skipping {} {} for {}", - &installer_language, + &data_language, system, - game.title); + content.title); continue; } @@ -395,11 +393,11 @@ impl<'a> Gog<'a> { for download in real_download { if !download.is_object() || !download.as_object().unwrap().contains_key("manualUrl") { - error!("Skipping an installer for {}", game.title); + error!("Skipping an installer for {}", content.title); continue; } - let installer = Installer { + let data = Data { manual_url: String::from(download["manualUrl"] .as_str() .unwrap()), @@ -407,10 +405,10 @@ impl<'a> Gog<'a> { .as_str() .unwrap_or("")), os: system.clone(), - language: installer_language.clone(), + language: data_language.clone(), }; - game.installers.push(installer); + content.data.push(data); } } } @@ -419,7 +417,7 @@ impl<'a> Gog<'a> { } } - Ok(game) + Ok(content) } fn auth_uri(&self, client_id: &str, redirect_uri: &str) -> String { @@ -455,8 +453,8 @@ impl<'a> Gog<'a> { refresh_token = refresh_token) } - fn game_uri(&self, game_id: u64) -> String { - format!("https://embed.gog.com/account/gameDetails/{game_id}.json", - game_id = game_id) + fn content_uri(&self, content_id: u64) -> String { + format!("https://embed.gog.com/account/gameDetails/{content_id}.json", + content_id = content_id) } } diff --git a/src/main.rs b/src/main.rs index fc9ca41..aafeff2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,9 +48,9 @@ fn main() { .version("0.2.2") .author("Sebastian Hugentobler ") .about("Synchronizes your gog library to a local folder.") - .arg(Arg::with_name("storage") + .arg(Arg::with_name("game-storage") .short("s") - .long("storage") + .long("game-storage") .value_name("FOLDER") .help("Sets the download folder (defaults to the working directory).") .takes_value(true)) @@ -58,7 +58,7 @@ fn main() { .short("m") .long("movie-storage") .value_name("FOLDER") - .help("Sets the download folder for movies (defaults to the game directory).") + .help("Sets the download folder for movies (defaults to the working directory).") .takes_value(true)) .arg(Arg::with_name("os") .short("o") @@ -91,14 +91,14 @@ fn main() { Err(_) => Config::new(), }; - let download_folder: &str = match matches.value_of("storage") { + let download_folder: &str = match matches.value_of("game-storage") { Some(value) => value, - None => config.storage.as_str(), + None => config.game_storage.as_str(), }; - let download_folder_movies: &str = match matches.value_of("movieStorage") { + let download_folder_movies: &str = match matches.value_of("movie-storage") { Some(value) => value, - None => download_folder, + None => config.movie_storage.as_str(), }; let os_filters: Vec = match matches.value_of("os") { diff --git a/src/models.rs b/src/models.rs index 2e3e43a..2cb0366 100644 --- a/src/models.rs +++ b/src/models.rs @@ -33,13 +33,13 @@ fn timestamp() -> i64 { #[serde(rename_all = "camelCase")] pub struct Config { #[serde(default)] - pub storage: String, + pub game_storage: String, #[serde(default)] pub movie_storage: String, #[serde(default = "default_map")] - pub games: HashMap, + pub content: HashMap, #[serde(default = "default_map")] - pub installers: HashMap, + pub data: HashMap, #[serde(default = "default_map")] pub extras: HashMap, #[serde(default = "default_list")] @@ -63,10 +63,10 @@ fn default_list() -> Vec { impl Config { pub fn new() -> Config { Config { - storage: String::from("."), + game_storage: String::from("."), movie_storage: String::from("."), - games: HashMap::new(), - installers: HashMap::new(), + content: HashMap::new(), + data: HashMap::new(), extras: HashMap::new(), os_filters: Vec::new(), language_filters: Vec::new(), @@ -79,17 +79,17 @@ impl Config { #[derive(Hash)] #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct Game { +pub struct Content { pub title: String, pub cd_key: String, #[serde(skip_deserializing)] pub is_movie: bool, #[serde(skip_deserializing)] - pub installers: Vec, + pub data: Vec, pub extras: Vec, } -impl fmt::Display for Game { +impl fmt::Display for Content { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "({})", self.title) } @@ -98,14 +98,14 @@ impl fmt::Display for Game { #[derive(Hash)] #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct Installer { +pub struct Data { pub manual_url: String, pub version: String, pub os: String, pub language: String, } -impl fmt::Display for Installer { +impl fmt::Display for Data { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "({}, {}, {}, {})", From 984aff32427dbf2151241a5bce6e7e63d0e6315c Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Wed, 22 Mar 2017 16:11:28 +0100 Subject: [PATCH 7/7] filter movies by resolution --- CHANGELOG.md | 1 + README.md | 18 ++++++++++++++++-- src/gog.rs | 34 ++++++++++++++++++++++++++++++---- src/main.rs | 29 ++++++++++++----------------- src/models.rs | 3 +++ 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 606fad1..88783e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.2.2 (2017-03-22) +- ability to filter movies by resolution - ability to save movies seperately - default value for storage config (fixes #6) - ability to skip movie and game content diff --git a/README.md b/README.md index b674676..41bf72b 100644 --- a/README.md +++ b/README.md @@ -46,11 +46,16 @@ A bare configuration with default values before first use: - *data*: A map, data url => hash - *extras*: A map, extra url => hash - *osFilters*: An array of operating systems. If it is not empty, game data is limited to the ones in the list. -Valid values are `linux`, `mac` and `windows` - *languageFilters*: An array of languages. If it is not empty, game data is limited to the ones in the list. +- *resolutionFilters*: An array of resolutions. If it is not empty, movie data is limited to the ones in the list. - *skipMovies*: Whether to skip movie content - *skipGames*: Whether to skip game content +Valid values for *osFilter*: +- `linux` +- `mac` +- `windows` + An incomplete list of languages on gog: - `english` - `český` @@ -63,6 +68,13 @@ An incomplete list of languages on gog: - `русский` - `中文` +An incomplete list of resolutions on gog: +- `DVD` +- `576p` +- `720p` +- `1080p` +- `4k` + You should have no need of changing `content`, `data` or `extras`, as these are used to determine whether specific content is up to date. @@ -71,7 +83,8 @@ used to determine whether specific content is up to date. If you want to see the information log while running set `RUST_LOG=info`. ``` -LAGS] [OPTIONS] +USAGE: + gog-sync [FLAGS] [OPTIONS] FLAGS: -h, --help Prints help information @@ -85,6 +98,7 @@ OPTIONS: -m, --movie-storage Sets the download folder for movies (defaults to the working directory). -o, --os Only sync files for this comma seperated list of operating systems. Valid values are 'linux', 'mac' and 'windows'. + -r, --resolution Only sync movies for this comma seperated list of resolutions. ``` --- diff --git a/src/gog.rs b/src/gog.rs index 6190fa7..f0baebe 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -125,6 +125,7 @@ impl<'a> Gog<'a> { storage_path_movies: &str, os_filters: &Vec, language_filters: &Vec, + resolution_filters: &Vec, skip_movies: bool, skip_games: bool) -> Result<(), GogError> { @@ -142,6 +143,7 @@ impl<'a> Gog<'a> { extras: HashMap::new(), os_filters: os_filters.clone(), language_filters: language_filters.clone(), + resolution_filters: resolution_filters.clone(), skip_movies: skip_movies, skip_games: skip_games, } @@ -156,7 +158,10 @@ impl<'a> Gog<'a> { None => u64::min_value(), }; - let content = match self.get_content(content_id, &os_filters, &language_filters) { + let content = match self.get_content(content_id, + &os_filters, + &language_filters, + &resolution_filters) { Ok(value) => value, Err(error) => { error!("{}: {}", &content_id, error); @@ -329,7 +334,8 @@ impl<'a> Gog<'a> { fn get_content(&mut self, content_id: u64, os_filters: &Vec, - language_filters: &Vec) + language_filters: &Vec, + resolution_filters: &Vec) -> Result { let content_uri = self.content_uri(content_id); debug!("looking for information at {}...", &content_uri); @@ -392,11 +398,31 @@ impl<'a> Gog<'a> { for real_download in real_downloads.as_array() { for download in real_download { if !download.is_object() || - !download.as_object().unwrap().contains_key("manualUrl") { - error!("Skipping an installer for {}", content.title); + !download.as_object().unwrap().contains_key("manualUrl") || + !download.as_object().unwrap().contains_key("name") { + error!("Skipping data for {}", content.title); continue; } + let name: &str = download["name"].as_str().unwrap(); + + + if content.is_movie && !resolution_filters.is_empty() { + let mut found_resolution = false; + for resolution_filter in resolution_filters { + let filter = format!("({})", resolution_filter); + if name.ends_with(&filter) { + found_resolution = true; + break; + } + } + + if !found_resolution { + info!("Skipping {}: not a suitable resolution.", name); + continue; + } + } + let data = Data { manual_url: String::from(download["manualUrl"] .as_str() diff --git a/src/main.rs b/src/main.rs index aafeff2..34488fe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,21 +1,4 @@ //! Synchronizes a [GOG](https://www.gog.com/) library with a local folder. -//! -//! ``` -//! USAGE: -//! gog-sync [OPTIONS] -//! -//! FLAGS: -//! -h, --help Prints help information -//! -V, --version Prints version information -//! -//! OPTIONS: -//! -l, --language Only sync files for this comma seperated list of languages. -//! -o, --os Only sync files for this comma seperated list of operating systems. -//! Valid values are linux, mac and windows. -//! -s, --storage Sets the download folder (defaults to the working directory). -//! ``` -//! -//! An incomplete list of languages on gog: `english, český, deutsch, español, français, italiano, magyar, polski, русский, 中文` extern crate chrono; extern crate clap; @@ -73,6 +56,12 @@ fn main() { .value_name("FILTER") .help("Only sync files for this comma seperated list of languages.") .takes_value(true)) + .arg(Arg::with_name("resolution") + .short("r") + .long("resolution") + .value_name("FILTER") + .help("Only sync movies for this comma seperated list of resolutions.") + .takes_value(true)) .arg(Arg::with_name("skip-movies") .short("f") .long("skip-movies") @@ -111,6 +100,11 @@ fn main() { None => config.language_filters, }; + let resolution_filters: Vec = match matches.value_of("resolution") { + Some(value) => value.split(',').map(String::from).collect(), + None => config.resolution_filters, + }; + let skip_movies = if matches.is_present("skip-movies") { true } else { @@ -130,6 +124,7 @@ fn main() { download_folder_movies, &os_filters, &language_filters, + &resolution_filters, skip_movies, skip_games) .unwrap(); diff --git a/src/models.rs b/src/models.rs index 2cb0366..2a07f0b 100644 --- a/src/models.rs +++ b/src/models.rs @@ -46,6 +46,8 @@ pub struct Config { pub os_filters: Vec, #[serde(default = "default_list")] pub language_filters: Vec, + #[serde(default = "default_list")] + pub resolution_filters: Vec, #[serde(default)] pub skip_movies: bool, #[serde(default)] @@ -70,6 +72,7 @@ impl Config { extras: HashMap::new(), os_filters: Vec::new(), language_filters: Vec::new(), + resolution_filters: Vec::new(), skip_movies: false, skip_games: false, }