diff --git a/.editorconfig b/.editorconfig index d741e40..fa9ce7c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,3 +7,6 @@ trim_trailing_whitespace = true insert_final_newline = true indent_style = space indent_size = 4 + +[*.md] +trim_trailing_whitespace = true diff --git a/CHANGELOG.md b/CHANGELOG.md index 5850b32..88783e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 0.2.2 (2017-03-22) +- ability to filter movies by resolution +- ability to save movies seperately +- default value for storage config (fixes #6) +- ability to skip movie and game content +- fix a bug with truncating an http response + ## 0.2.1 (2017-03-21) - add instructions for getting the code if there is no token diff --git a/Cargo.lock b/Cargo.lock index 5787265..0a50d85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ [root] name = "gog-sync" -version = "0.2.0" +version = "0.2.2" dependencies = [ "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index c3c9ae2..5d33ed1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gog-sync" -version = "0.2.1" +version = "0.2.2" authors = ["Sebastian Hugentobler "] description = "Synchronizes a GOG library with a local folder." documentation = "https://docs.rs/crate/gog-sync" diff --git a/README.md b/README.md index 1deb100..41bf72b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # GOG-SYNC +## **Please note that this is alpha software, you should not trust it.** A small tool to synchronize the stuff in a [GOG](https://www.gog.com/) library with a local folder. @@ -7,8 +8,6 @@ It builds on the work of the [unofficial GOG API Documentation](https://gogapido This is the first time I am building something with rust, so beware :) -**Please note that this is alpha software, you should not trust it.** - # Installation Install from [crates.io](https://crates.io). @@ -26,13 +25,82 @@ For example on macOS or Linux ~/.config/gog-sync/config.json ``` -It is in Json format and the only relevant key is `storage`. The rest is information -about content hashes. +A bare configuration with default values before first use: +``` +{ + "gameStorage": ".", + "movieStorage": ".", + "content": {}, + "data": {}, + "extras": {}, + "osFilters": [], + "languageFilters": [], + "skipMovies": false, + "skipGames": false +} +``` + +- *gameStorage*: Where to save games +- *movieStorage*: Where to save movies +- *content*: A map, content id => hash +- *data*: A map, data url => hash +- *extras*: A map, extra url => hash +- *osFilters*: An array of operating systems. If it is not empty, game data is limited to the ones in the list. +- *languageFilters*: An array of languages. If it is not empty, game data is limited to the ones in the list. +- *resolutionFilters*: An array of resolutions. If it is not empty, movie data is limited to the ones in the list. +- *skipMovies*: Whether to skip movie content +- *skipGames*: Whether to skip game content + +Valid values for *osFilter*: +- `linux` +- `mac` +- `windows` + +An incomplete list of languages on gog: +- `english` +- `český` +- `deutsch` +- `español` +- `français` +- `italiano` +- `magyar` +- `polski` +- `русский` +- `中文` + +An incomplete list of resolutions on gog: +- `DVD` +- `576p` +- `720p` +- `1080p` +- `4k` + +You should have no need of changing `content`, `data` or `extras`, as these are +used to determine whether specific content is up to date. # Usage If you want to see the information log while running set `RUST_LOG=info`. +``` +USAGE: + gog-sync [FLAGS] [OPTIONS] + +FLAGS: + -h, --help Prints help information + -g, --skip-games Skip game content. + -f, --skip-movies Skip movie content. + -V, --version Prints version information + +OPTIONS: + -s, --game-storage Sets the download folder (defaults to the working directory). + -l, --language Only sync files for this comma seperated list of languages. + -m, --movie-storage Sets the download folder for movies (defaults to the working directory). + -o, --os Only sync files for this comma seperated list of operating systems. + Valid values are 'linux', 'mac' and 'windows'. + -r, --resolution Only sync movies for this comma seperated list of resolutions. +``` + --- ``` diff --git a/src/configfiles.rs b/src/configfiles.rs index 6eb1ba9..2915a32 100644 --- a/src/configfiles.rs +++ b/src/configfiles.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_json; +use std::fmt; use std::fs::File; use std; use std::io::{Read, Write}; @@ -20,6 +21,15 @@ pub enum ConfigError { SerdeJsonError(serde_json::Error), } +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + ConfigError::IOError(ref err) => fmt::Display::fmt(err, f), + ConfigError::SerdeJsonError(ref err) => fmt::Display::fmt(err, f), + } + } +} + impl From for ConfigError { fn from(e: std::io::Error) -> Self { ConfigError::IOError(e) diff --git a/src/gog.rs b/src/gog.rs index a9bead7..f0baebe 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -13,10 +13,11 @@ use configfiles::{ConfigFiles, ConfigError}; use http::{Http, HttpError}; use models; -use models::{Token, Game, Installer, Config}; +use models::{Token, Content, Data, Config}; use serde_json; use serde_json::Value; use std::collections::HashMap; +use std::fmt; use std::fs; use std::fs::File; use std::io; @@ -33,6 +34,18 @@ pub enum GogError { IOError(io::Error), } +impl fmt::Display for GogError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + GogError::Error(ref err) => fmt::Display::fmt(err, f), + GogError::ConfigError(ref err) => fmt::Display::fmt(err, f), + GogError::HttpError(ref err) => fmt::Display::fmt(err, f), + GogError::SerdeError(ref err) => fmt::Display::fmt(err, f), + GogError::IOError(ref err) => fmt::Display::fmt(err, f), + } + } +} + impl From for GogError { fn from(e: ConfigError) -> Self { GogError::ConfigError(e) @@ -109,8 +122,12 @@ impl<'a> Gog<'a> { /// Uses a hash to figure out whether something has changed. pub fn sync(&mut self, storage_path: &str, + storage_path_movies: &str, os_filters: &Vec, - language_filters: &Vec) + language_filters: &Vec, + resolution_filters: &Vec, + skip_movies: bool, + skip_games: bool) -> Result<(), GogError> { let configfiles = ConfigFiles::new(); let mut config: Config = match configfiles.load("config.json") { @@ -119,68 +136,90 @@ impl<'a> Gog<'a> { error!("Configuration error, generating new one..."); Config { - storage: String::from(storage_path), - games: HashMap::new(), - installers: HashMap::new(), + game_storage: String::from(storage_path), + movie_storage: String::from(storage_path_movies), + content: HashMap::new(), + data: HashMap::new(), extras: HashMap::new(), os_filters: os_filters.clone(), language_filters: language_filters.clone(), + resolution_filters: resolution_filters.clone(), + skip_movies: skip_movies, + skip_games: skip_games, } } }; - let game_ids = self.get_game_ids()?; + let content_ids = self.get_content_ids()?; - for game_id in game_ids { - let game_hash_saved = match config.games.get(&game_id.to_string()) { + for content_id in content_ids { + let content_hash_saved = match config.content.get(&content_id.to_string()) { Some(value) => value.clone(), None => u64::min_value(), }; - let game = self.get_game(game_id, &os_filters, &language_filters)?; - let game_hash = models::get_hash(&game); + let content = match self.get_content(content_id, + &os_filters, + &language_filters, + &resolution_filters) { + Ok(value) => value, + Err(error) => { + error!("{}: {}", &content_id, error); + continue; + } + }; - if game_hash_saved == game_hash { - info!("{} already up to date.", &game.title); + if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { + info!("filtering {}", content.title); continue; } - let game_root = Path::new(storage_path).join(&game.title); - fs::create_dir_all(&game_root)?; + let content_hash = models::get_hash(&content); - if !game.cd_key.is_empty() { - let key_path = Path::new(game_root.as_os_str()).join("key.txt"); - let mut key_file = File::create(&key_path)?; - key_file.write_all(game.cd_key.as_bytes())? + if content_hash_saved == content_hash { + info!("{} already up to date.", &content.title); + continue; } - for installer in game.installers { - let installer_hash_saved = match config.installers.get(&installer.manual_url) { + let content_root = if content.is_movie { + Path::new(storage_path_movies).join(&content.title) + } else { + Path::new(storage_path).join(&content.title) + }; + + fs::create_dir_all(&content_root)?; + + if !content.cd_key.is_empty() { + let key_path = Path::new(content_root.as_os_str()).join("key.txt"); + let mut key_file = File::create(&key_path)?; + key_file.write_all(content.cd_key.as_bytes())? + } + + for data in content.data { + let data_hash_saved = match config.data.get(&data.manual_url) { Some(value) => value.clone(), None => u64::min_value(), }; - let installer_hash = models::get_hash(&installer); + let data_hash = models::get_hash(&data); - if installer_hash_saved == installer_hash { - info!("{} already up to date.", &installer.manual_url); + if data_hash_saved == data_hash { + info!("{} already up to date.", &data.manual_url); continue; } - let installer_root = Path::new(&game_root).join(&installer.language); - fs::create_dir_all(&installer_root)?; + let data_root = Path::new(&content_root).join(&data.language); + fs::create_dir_all(&data_root)?; - let installer_uri = format!("https://embed.gog.com{}", installer.manual_url); + let data_uri = format!("https://embed.gog.com{}", data.manual_url); - info!("downloading {} for {}...", - &installer.manual_url, - &game.title); - self.http_client.download(installer_uri.as_str(), &installer_root)?; + info!("downloading {} for {}...", &data.manual_url, &content.title); + self.http_client.download(data_uri.as_str(), &data_root)?; - config.installers.insert(installer.manual_url, installer_hash); + config.data.insert(data.manual_url, data_hash); configfiles.save("config.json", &config)?; } - for extra in game.extras { + for extra in content.extras { let extra_hash_saved = match config.extras.get(&extra.manual_url) { Some(value) => value.clone(), None => u64::min_value(), @@ -195,14 +234,14 @@ impl<'a> Gog<'a> { let extra_uri = format!("https://embed.gog.com{}", extra.manual_url); - info!("downloading {} for {}...", &extra.name, &game.title); - self.http_client.download(extra_uri.as_str(), &game_root)?; + info!("downloading {} for {}...", &extra.name, &content.title); + self.http_client.download(extra_uri.as_str(), &content_root)?; config.extras.insert(extra.manual_url, extra_hash); configfiles.save("config.json", &config)?; } - config.games.insert(game_id.to_string(), game_hash); + config.content.insert(content_id.to_string(), content_hash); configfiles.save("config.json", &config)?; } @@ -261,80 +300,97 @@ impl<'a> Gog<'a> { } } - fn get_game_ids(&mut self) -> Result, GogError> { + fn get_content_ids(&mut self) -> Result, GogError> { let response = self.http_client.get(self.games_uri.as_str())?; - let game_ids_raw: Value = serde_json::from_str(response.as_str())?; - let game_ids_serde = &game_ids_raw["owned"]; + let content_ids_raw: Value = serde_json::from_str(response.as_str())?; + let content_ids_serde = &content_ids_raw["owned"]; - let mut game_ids: Vec = Vec::new(); + let mut content_ids: Vec = Vec::new(); - if !game_ids_serde.is_array() { - return Err(GogError::Error("Error parsing game ids.")); + if !content_ids_serde.is_array() { + return Err(GogError::Error("Error parsing content ids.")); } - for game_id in game_ids_serde.as_array().unwrap() { - let game_id_parsed = game_id.as_u64().unwrap_or(0); + for content_id in content_ids_serde.as_array().unwrap() { + let content_id_parsed = content_id.as_u64().unwrap_or(0); - if game_id_parsed == 0 { - error!("Cant parse game id {}", game_id); + if content_id_parsed == 0 { + error!("Cant parse content id {}", content_id); continue; } // The numbers in this list are excluded because they refer to // favourites, promotions and such. - if [1, 2, 3, 4, 5].contains(&game_id_parsed) { + if [1, 2, 3, 4, 5].contains(&content_id_parsed) { continue; } - game_ids.push(game_id_parsed); + content_ids.push(content_id_parsed); } - return Ok(game_ids); + return Ok(content_ids); } - fn get_game(&mut self, - game_id: u64, - os_filters: &Vec, - language_filters: &Vec) - -> Result { - let game_uri = self.game_uri(game_id); + fn get_content(&mut self, + content_id: u64, + os_filters: &Vec, + language_filters: &Vec, + resolution_filters: &Vec) + -> Result { + let content_uri = self.content_uri(content_id); + debug!("looking for information at {}...", &content_uri); - let response = self.http_client.get(game_uri.as_str())?; + let response = self.http_client.get(content_uri.as_str())?; - let mut game: Game = serde_json::from_str(&response)?; + let content_raw: Value = serde_json::from_str(response.as_str())?; + debug!("found {:?}", &content_raw); - let game_raw: Value = serde_json::from_str(response.as_str())?; - let downloads = &game_raw["downloads"]; + let mut content: Content = serde_json::from_str(&response)?; + let downloads = &content_raw["downloads"]; + + if content_raw.is_object() && !content_raw.as_object().unwrap().contains_key("forumLink") { + return Err(GogError::Error("No forumLink property")); + } + + let is_movie = match content_raw["forumLink"].as_str() { + Some(value) => value == "https://embed.gog.com/forum/movies", + None => false, + }; + + content.is_movie = is_movie; + + debug!("processing installer fields: {:?}", &downloads); for languages in downloads.as_array() { for language in languages { if !language.is_array() || language.as_array().unwrap().len() < 2 { - error!("Skipping a language for {}", game.title); + error!("Skipping a language for {}", content.title); continue; } - let installer_language = match language[0].as_str() { + let data_language = match language[0].as_str() { Some(value) => value.to_lowercase(), None => String::default(), }; - if installer_language.is_empty() { - error!("Skipping a language for {}", game.title); + if data_language.is_empty() { + error!("Skipping a language for {}", content.title); continue; } - if !language_filters.is_empty() && !language_filters.contains(&installer_language) { - info!("Skipping {} for {}", &installer_language, game.title); + if !is_movie && !language_filters.is_empty() && + !language_filters.contains(&data_language) { + info!("Skipping {} for {}", &data_language, content.title); continue; } for systems in language[1].as_object() { for system in systems.keys() { - if !os_filters.is_empty() && !os_filters.contains(system) { + if is_movie && !os_filters.is_empty() && !os_filters.contains(system) { info!("Skipping {} {} for {}", - &installer_language, + &data_language, system, - game.title); + content.title); continue; } @@ -342,12 +398,32 @@ impl<'a> Gog<'a> { for real_download in real_downloads.as_array() { for download in real_download { if !download.is_object() || - !download.as_object().unwrap().contains_key("manualUrl") { - error!("Skipping an installer for {}", game.title); + !download.as_object().unwrap().contains_key("manualUrl") || + !download.as_object().unwrap().contains_key("name") { + error!("Skipping data for {}", content.title); continue; } - let installer = Installer { + let name: &str = download["name"].as_str().unwrap(); + + + if content.is_movie && !resolution_filters.is_empty() { + let mut found_resolution = false; + for resolution_filter in resolution_filters { + let filter = format!("({})", resolution_filter); + if name.ends_with(&filter) { + found_resolution = true; + break; + } + } + + if !found_resolution { + info!("Skipping {}: not a suitable resolution.", name); + continue; + } + } + + let data = Data { manual_url: String::from(download["manualUrl"] .as_str() .unwrap()), @@ -355,10 +431,10 @@ impl<'a> Gog<'a> { .as_str() .unwrap_or("")), os: system.clone(), - language: installer_language.clone(), + language: data_language.clone(), }; - game.installers.push(installer); + content.data.push(data); } } } @@ -367,7 +443,7 @@ impl<'a> Gog<'a> { } } - Ok(game) + Ok(content) } fn auth_uri(&self, client_id: &str, redirect_uri: &str) -> String { @@ -403,8 +479,8 @@ impl<'a> Gog<'a> { refresh_token = refresh_token) } - fn game_uri(&self, game_id: u64) -> String { - format!("https://embed.gog.com/account/gameDetails/{game_id}.json", - game_id = game_id) + fn content_uri(&self, content_id: u64) -> String { + format!("https://embed.gog.com/account/gameDetails/{content_id}.json", + content_id = content_id) } } diff --git a/src/http.rs b/src/http.rs index 5fdd8c2..a4b40ca 100644 --- a/src/http.rs +++ b/src/http.rs @@ -3,6 +3,7 @@ use curl; use curl::easy::{Easy, List, WriteError}; +use std::fmt; use std::fs; use std::fs::File; use std::io; @@ -24,6 +25,18 @@ pub enum HttpError { UrlParseError(url::ParseError), } +impl fmt::Display for HttpError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + HttpError::Error(ref err) => fmt::Display::fmt(err, f), + HttpError::CurlError(ref err) => fmt::Display::fmt(err, f), + HttpError::Utf8Error(ref err) => fmt::Display::fmt(err, f), + HttpError::IOError(ref err) => fmt::Display::fmt(err, f), + HttpError::UrlParseError(ref err) => fmt::Display::fmt(err, f), + } + } +} + impl From for HttpError { fn from(e: curl::Error) -> Self { HttpError::CurlError(e) @@ -79,20 +92,22 @@ impl Http { /// http_client.get("https://discworld.com/"); /// ``` pub fn get(&mut self, uri: &str) -> Result { - let mut response_body = String::new(); - + let mut data = Vec::new(); self.curl.url(uri)?; { let mut transfer = self.curl.transfer(); - transfer.write_function(|data| { - response_body = String::from(str::from_utf8(data).unwrap()); - Ok(data.len()) + transfer.write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) })?; transfer.perform()?; } - Ok(response_body) + match str::from_utf8(data.as_slice()) { + Ok(value) => Ok(value.to_owned()), + Err(error) => Err(HttpError::Utf8Error(error)), + } } /// Add a header to all future requests. diff --git a/src/main.rs b/src/main.rs index 442079a..34488fe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,19 +1,4 @@ //! Synchronizes a [GOG](https://www.gog.com/) library with a local folder. -//! -//! ``` -//! USAGE: -//! gog-sync [OPTIONS] -//! -//! FLAGS: -//! -h, --help Prints help information -//! -V, --version Prints version information -//! -//! OPTIONS: -//! -l, --language Only sync files for this comma seperated list of languages. -//! -o, --os Only sync files for this comma seperated list of operating systems. -//! Valid values are 'linux', 'mac' and 'windows'. -//! -s, --storage Sets the download folder (defaults to the working directory). -//! ``` extern crate chrono; extern crate clap; @@ -43,15 +28,21 @@ fn main() { env_logger::init().unwrap(); let matches = App::new("Gog Synchronizer") - .version("0.2.1") + .version("0.2.2") .author("Sebastian Hugentobler ") .about("Synchronizes your gog library to a local folder.") - .arg(Arg::with_name("storage") + .arg(Arg::with_name("game-storage") .short("s") - .long("storage") + .long("game-storage") .value_name("FOLDER") .help("Sets the download folder (defaults to the working directory).") .takes_value(true)) + .arg(Arg::with_name("movie-storage") + .short("m") + .long("movie-storage") + .value_name("FOLDER") + .help("Sets the download folder for movies (defaults to the working directory).") + .takes_value(true)) .arg(Arg::with_name("os") .short("o") .long("os") @@ -65,6 +56,22 @@ fn main() { .value_name("FILTER") .help("Only sync files for this comma seperated list of languages.") .takes_value(true)) + .arg(Arg::with_name("resolution") + .short("r") + .long("resolution") + .value_name("FILTER") + .help("Only sync movies for this comma seperated list of resolutions.") + .takes_value(true)) + .arg(Arg::with_name("skip-movies") + .short("f") + .long("skip-movies") + .help("Skip movie content.") + .takes_value(false)) + .arg(Arg::with_name("skip-games") + .short("g") + .long("skip-games") + .help("Skip game content.") + .takes_value(false)) .get_matches(); let configfiles = ConfigFiles::new(); @@ -73,9 +80,14 @@ fn main() { Err(_) => Config::new(), }; - let download_folder: &str = match matches.value_of("storage") { + let download_folder: &str = match matches.value_of("game-storage") { Some(value) => value, - None => config.storage.as_str(), + None => config.game_storage.as_str(), + }; + + let download_folder_movies: &str = match matches.value_of("movie-storage") { + Some(value) => value, + None => config.movie_storage.as_str(), }; let os_filters: Vec = match matches.value_of("os") { @@ -88,8 +100,32 @@ fn main() { None => config.language_filters, }; + let resolution_filters: Vec = match matches.value_of("resolution") { + Some(value) => value.split(',').map(String::from).collect(), + None => config.resolution_filters, + }; + + let skip_movies = if matches.is_present("skip-movies") { + true + } else { + config.skip_movies + }; + + let skip_games = if matches.is_present("skip-games") { + true + } else { + config.skip_games + }; + let mut http_client = Http::new(); let mut gog = Gog::new(&mut http_client); gog.login().unwrap(); - gog.sync(download_folder, &os_filters, &language_filters).unwrap(); + gog.sync(download_folder, + download_folder_movies, + &os_filters, + &language_filters, + &resolution_filters, + skip_movies, + skip_games) + .unwrap(); } diff --git a/src/models.rs b/src/models.rs index fd29581..2a07f0b 100644 --- a/src/models.rs +++ b/src/models.rs @@ -33,17 +33,25 @@ fn timestamp() -> i64 { #[serde(rename_all = "camelCase")] pub struct Config { #[serde(default)] - pub storage: String, + pub game_storage: String, + #[serde(default)] + pub movie_storage: String, #[serde(default = "default_map")] - pub games: HashMap, + pub content: HashMap, #[serde(default = "default_map")] - pub installers: HashMap, + pub data: HashMap, #[serde(default = "default_map")] pub extras: HashMap, #[serde(default = "default_list")] pub os_filters: Vec, #[serde(default = "default_list")] pub language_filters: Vec, + #[serde(default = "default_list")] + pub resolution_filters: Vec, + #[serde(default)] + pub skip_movies: bool, + #[serde(default)] + pub skip_games: bool, } fn default_map() -> HashMap { @@ -57,12 +65,16 @@ fn default_list() -> Vec { impl Config { pub fn new() -> Config { Config { - storage: String::from("."), - games: HashMap::new(), - installers: HashMap::new(), + game_storage: String::from("."), + movie_storage: String::from("."), + content: HashMap::new(), + data: HashMap::new(), extras: HashMap::new(), os_filters: Vec::new(), language_filters: Vec::new(), + resolution_filters: Vec::new(), + skip_movies: false, + skip_games: false, } } } @@ -70,15 +82,17 @@ impl Config { #[derive(Hash)] #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct Game { +pub struct Content { pub title: String, pub cd_key: String, #[serde(skip_deserializing)] - pub installers: Vec, + pub is_movie: bool, + #[serde(skip_deserializing)] + pub data: Vec, pub extras: Vec, } -impl fmt::Display for Game { +impl fmt::Display for Content { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "({})", self.title) } @@ -87,14 +101,14 @@ impl fmt::Display for Game { #[derive(Hash)] #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct Installer { +pub struct Data { pub manual_url: String, pub version: String, pub os: String, pub language: String, } -impl fmt::Display for Installer { +impl fmt::Display for Data { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "({}, {}, {}, {})",