From 7a40242d67dabe52eef0501fea56d428bd6f63d5 Mon Sep 17 00:00:00 2001 From: Sebastian Hugentobler Date: Wed, 10 May 2017 14:19:09 +0200 Subject: [PATCH] save dlcs --- src/gog.rs | 254 ++++++++++++++++++++++++-------------- src/models/content.rs | 19 +++ src/sync.rs | 277 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 461 insertions(+), 89 deletions(-) create mode 100644 src/sync.rs diff --git a/src/gog.rs b/src/gog.rs index 7172156..6361dae 100644 --- a/src/gog.rs +++ b/src/gog.rs @@ -15,15 +15,18 @@ use http::{Http, HttpError}; use models; use models::content::Content; use models::contentinfo::ContentInfo; +use models::data::Data; use models::datainfo::DataInfo; +use models::extra::Extra; use models::extrainfo::ExtraInfo; use models::token::Token; use serde_json; use serde_json::Value; +use std::collections::BTreeMap; use std::collections::HashMap; -use std::fmt; use std::fs; use std::fs::File; +use std::fmt; use std::io; use std::io::Write; use std::path::Path; @@ -131,7 +134,7 @@ impl<'a> Gog<'a> { /// Syncs the contents of a gog account with a local folder. /// Uses a hash to figure out whether something has changed. pub fn sync(&mut self, - storage_path: &str, + storage_path_games: &str, storage_path_movies: &str, os_filters: &Vec, language_filters: &Vec, @@ -153,120 +156,192 @@ impl<'a> Gog<'a> { } }; - let content_root = if content.is_movie { - Path::new(storage_path_movies).join(&content.title) - } else { - Path::new(storage_path).join(&content.title) - }; - - let content_info_path = Path::new(&content_root).join("info.json"); - - let content_info_saved = - match ConfigFiles::load_from_path::(&content_info_path) { - Ok(value) => value, - Err(_) => ContentInfo::new(), - }; - if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { info!("filtering {}", content.title); continue; } - let content_hash = models::get_hash(&content); - - if content_info_saved.hash == content_hash { - info!("{} already up to date.", &content.title); - continue; + match self.sync_content(content, storage_path_games, storage_path_movies) { + Ok(_) => (), + Err(error) => { + warn!("{:?}", error); + continue; + } } + } + Ok(()) + } - fs::create_dir_all(&content_root)?; + fn sync_content(&mut self, + content: Content, + storage_path_games: &str, + storage_path_movies: &str) + -> Result<(), GogError> { - let mut content_info = ContentInfo { - hash: content_hash, - id: content_id, - title: content.title.clone(), - cd_keys: content.cd_keys.clone(), - data: HashMap::new(), - extras: HashMap::new(), + let content_root = if content.is_movie { + Path::new(storage_path_movies).join(&content.title) + } else { + Path::new(storage_path_games).join(&content.title) + }; + + let content_info_path = Path::new(&content_root).join("info.json"); + let content_info_saved = + match ConfigFiles::load_from_path::(&content_info_path) { + Ok(value) => value, + Err(_) => ContentInfo::new(), }; - ConfigFiles::save_to_path(&content_info_path, &content_info)?; + let content_hash = models::get_hash(&content); - let key_root = content_root.join("keys"); + if content_info_saved.hash == content_hash { + info!("{} already up to date.", &content.title); + return Ok(()); + } - if content.cd_keys.len() > 0 { - fs::create_dir_all(&key_root)?; - } + fs::create_dir_all(&content_root)?; - for (key, value) in content.cd_keys { - let key_path = key_root.join(format!("{}.txt", key)); - let mut key_file = File::create(&key_path)?; - key_file.write_all(value.as_bytes())? - } + let mut content_info = ContentInfo { + hash: content_hash, + id: content.id, + title: content.title.clone(), + cd_keys: content.cd_keys.clone(), + data: HashMap::new(), + extras: HashMap::new(), + }; - for data in content.data { - let data_hash_saved = match content_info_saved.data.get(&content_id.to_string()) { - Some(value) => value.hash, - None => u64::min_value(), - }; + ConfigFiles::save_to_path(&content_info_path, &content_info)?; - let data_hash = models::get_hash(&data); + self.save_keys(content.cd_keys, &content_root)?; - if data_hash_saved == data_hash { - info!("{} already up to date.", &data.manual_url); - continue; - } + for data in content.data { + self.save_data(content.title.as_str(), + data, + &content_root, + &content_info_path, + &content_info_saved, + &mut content_info)?; + } - let data_root = Path::new(&content_root).join(&data.language); - fs::create_dir_all(&data_root)?; + let dlc_root = Path::new(&storage_path_games) + .join(&content.title) + .join("dlcs"); - let data_uri = format!("https://embed.gog.com{}", data.manual_url); + for dlc in content.dlcs { + self.sync_content(dlc, + dlc_root.to_string_lossy().as_ref(), + storage_path_movies)?; + } - info!("downloading {} for {}...", &data.manual_url, &content.title); - let filename = self.api_request_download(data_uri.as_str(), &data_root)?; + for extra in content.extras { + self.save_extra(content.title.as_str(), + extra, + &content_root, + &content_info_path, + &content_info_saved, + &mut content_info)?; + } - let data_info = DataInfo { - hash: data_hash, - filename: filename, - language: data.language, - }; + Ok(()) + } - content_info - .data - .insert(data.manual_url.clone(), data_info); - ConfigFiles::save_to_path(&content_info_path, &content_info)?; - } + fn save_file(&mut self, + content_title: &str, + relative_uri: &str, + data_root: &PathBuf) + -> Result { + fs::create_dir_all(&data_root)?; + let data_uri = format!("https://embed.gog.com{}", relative_uri); - for extra in content.extras { - let extra_hash_saved = - match content_info_saved.extras.get(&content_id.to_string()) { - Some(value) => value.hash, - None => u64::min_value(), - }; + info!("downloading {} for {}...", &relative_uri, content_title); + self.api_request_download(data_uri.as_str(), &data_root) + } - let extra_hash = models::get_hash(&extra); + fn save_data(&mut self, + content_title: &str, + data: Data, + content_root: &PathBuf, + content_info_path: &PathBuf, + content_info_saved: &ContentInfo, + content_info: &mut ContentInfo) + -> Result<(), GogError> { + let hash_saved = match content_info_saved.data.get(data.manual_url.as_str()) { + Some(value) => value.hash, + None => u64::min_value(), + }; - if extra_hash_saved == extra_hash { - info!("{} already up to date.", &extra.manual_url); - continue; - } + let hash = models::get_hash(&data); + if hash_saved == hash { + info!("{} already up to date.", &data.manual_url); + return Ok(()); + } - let extra_uri = format!("https://embed.gog.com{}", extra.manual_url); + let data_root = Path::new(&content_root).join(&data.language); - info!("downloading {} for {}...", &extra.name, &content.title); - let filename = self.api_request_download(extra_uri.as_str(), &content_root)?; + let filename = self.save_file(content_title, &data.manual_url, &data_root)?; - let extra_info = ExtraInfo { - hash: extra_hash, - filename: filename, - name: extra.name, - }; + let data_info = DataInfo { + hash: hash, + filename: filename, + language: data.language, + }; - content_info - .extras - .insert(extra.manual_url.clone(), extra_info); - ConfigFiles::save_to_path(&content_info_path, &content_info)?; - } + content_info + .data + .insert(data.manual_url.clone(), data_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + + Ok(()) + } + + fn save_extra(&mut self, + content_title: &str, + extra: Extra, + content_root: &PathBuf, + content_info_path: &PathBuf, + content_info_saved: &ContentInfo, + content_info: &mut ContentInfo) + -> Result<(), GogError> { + let hash_saved = match content_info_saved.data.get(extra.manual_url.as_str()) { + Some(value) => value.hash, + None => u64::min_value(), + }; + + let hash = models::get_hash(&extra); + if hash_saved == hash { + info!("{} already up to date.", &extra.manual_url); + return Ok(()); + } + + let filename = self.save_file(content_title, &extra.manual_url, &content_root)?; + + let extra_info = ExtraInfo { + hash: hash, + filename: filename, + name: extra.name, + }; + + content_info + .extras + .insert(extra.manual_url.clone(), extra_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + + Ok(()) + } + + fn save_keys(&mut self, + cd_keys: BTreeMap, + content_root: &PathBuf) + -> Result<(), GogError> { + let key_root = content_root.join("keys"); + + if cd_keys.len() > 0 { + fs::create_dir_all(&key_root)?; + } + + for (key, value) in cd_keys { + let key_path = key_root.join(format!("{}.txt", key)); + let mut key_file = File::create(&key_path)?; + key_file.write_all(value.as_bytes())? } Ok(()) @@ -347,7 +422,8 @@ impl<'a> Gog<'a> { content_ids.push(content_id_parsed); } - return Ok(content_ids); + //Ok(content_ids) + Ok(vec![1207658995]) } fn api_request_ensure_token(&mut self, response: &str) -> Result { diff --git a/src/models/content.rs b/src/models/content.rs index 2148d8c..b8a506d 100644 --- a/src/models/content.rs +++ b/src/models/content.rs @@ -102,6 +102,7 @@ pub fn deserialize(content_id: u64, let mut content: Content = serde_json::from_str(content_string)?; content.id = content_id; + let dlcs = &content_raw["dlcs"]; let downloads = &content_raw["downloads"]; if content_raw.is_object() && @@ -124,6 +125,24 @@ pub fn deserialize(content_id: u64, content.cd_keys = deserialize_cd_keys(&content.title, cd_keys_raw); } + match dlcs.as_array() { + Some(value) => { + if value.len() > 0 { + debug!("processing dlc fields: {:?}", &value); + for dlc in value { + let dlc = deserialize(content_id, + serde_json::to_string(&dlc).unwrap().as_str(), + os_filters, + language_filters, + resolution_filters)?; + content.dlcs.push(dlc); + } + } + } + None => (), + } + + debug!("processing installer fields: {:?}", &downloads); for languages in downloads.as_array() { for language in languages { diff --git a/src/sync.rs b/src/sync.rs new file mode 100644 index 0000000..74e2886 --- /dev/null +++ b/src/sync.rs @@ -0,0 +1,277 @@ +use configfiles::ConfigFiles; +use gog::{Gog, GogError}; +use http::{Http, HttpError}; +use models; +use models::content::Content; +use models::contentinfo::ContentInfo; +use models::data::Data; +use models::datainfo::DataInfo; +use models::extra::Extra; +use std::collections::HashMap; +use std::collections::BTreeMap; +use std::io; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::fs; +use std::fs::File; + +pub fn sync_content(content: Content, + storage_path_games: &str, + storage_path_movies: &str, + os_filters: &Vec, + language_filters: &Vec, + resolution_filters: &Vec, + skip_movies: bool, + skip_games: bool) + -> Result<(), GogError> { + + let content_root = if content.is_movie { + Path::new(storage_path_movies).join(&content.title) + } else { + Path::new(storage_path_games).join(&content.title) + }; + + let content_info_path = Path::new(&content_root).join("info.json"); + let content_info_saved = match ConfigFiles::load_from_path::(&content_info_path) { + Ok(value) => value, + Err(_) => ContentInfo::new(), + }; + + if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { + info!("filtering {}", content.title); + return Ok(()); + } + + let content_hash = models::get_hash(&content); + + if content_info_saved.hash == content_hash { + info!("{} already up to date.", &content.title); + return Ok(()); + } + + fs::create_dir_all(&content_root)?; + + let mut content_info = ContentInfo { + hash: content_hash, + id: content.id, + title: content.title.clone(), + cd_keys: content.cd_keys.clone(), + data: HashMap::new(), + extras: HashMap::new(), + }; + + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + + save_keys(content.cd_keys, &content_root)?; + + for data in content.data { + save_data(content.title.as_str(), + data, + &content_root, + &content_info_path, + &content_info_saved, + &mut content_info); + } + + Ok(()) +} + +fn save_data(content_title: &str, + data: Data, + content_root: &PathBuf, + content_info_path: &PathBuf, + content_info_saved: &ContentInfo, + content_info: &mut ContentInfo) + -> Result<(), GogError> { + let data_hash_saved = match content_info_saved.data.get(data.manual_url.as_str()) { + Some(value) => value.hash, + None => u64::min_value(), + }; + + let data_hash = models::get_hash(&data); + if data_hash_saved == data_hash { + info!("{} already up to date.", &data.manual_url); + return Ok(()); + } + + let data_root = Path::new(&content_root).join(&data.language); + fs::create_dir_all(&data_root)?; + + let data_uri = format!("https://embed.gog.com{}", data.manual_url); + + info!("downloading {} for {}...", &data.manual_url, content_title); + let filename = self.api_request_download(data_uri.as_str(), &data_root)?; + + let data_info = DataInfo { + hash: data_hash, + filename: filename, + language: data.language, + }; + + content_info + .data + .insert(data.manual_url.clone(), data_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + + Ok(()) +} + +fn save_keys(cd_keys: BTreeMap, content_root: &PathBuf) -> Result<(), GogError> { + let key_root = content_root.join("keys"); + + if cd_keys.len() > 0 { + fs::create_dir_all(&key_root)?; + } + + for (key, value) in cd_keys { + let key_path = key_root.join(format!("{}.txt", key)); + let mut key_file = File::create(&key_path)?; + key_file.write_all(value.as_bytes())? + } + + Ok(()) +} + +/* +/// Syncs the contents of a gog account with a local folder. +/// Uses a hash to figure out whether something has changed. +pub fn syncold(storage_path: &str, + storage_path_movies: &str, + os_filters: &Vec, + language_filters: &Vec, + resolution_filters: &Vec, + skip_movies: bool, + skip_games: bool) + -> Result<(), GogError> { + let content_ids = self.get_content_ids()?; + + for content_id in content_ids { + let content = match self.get_content(content_id, + &os_filters, + &language_filters, + &resolution_filters) { + Ok(value) => value, + Err(error) => { + error!("{}: {}", &content_id, error); + continue; + } + }; + + let content_root = if content.is_movie { + Path::new(storage_path_movies).join(&content.title) + } else { + Path::new(storage_path).join(&content.title) + }; + + let content_info_path = Path::new(&content_root).join("info.json"); + + let content_info_saved = + match ConfigFiles::load_from_path::(&content_info_path) { + Ok(value) => value, + Err(_) => ContentInfo::new(), + }; + + if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { + info!("filtering {}", content.title); + continue; + } + + let content_hash = models::get_hash(&content); + + if content_info_saved.hash == content_hash { + info!("{} already up to date.", &content.title); + continue; + } + + fs::create_dir_all(&content_root)?; + + let mut content_info = ContentInfo { + hash: content_hash, + id: content_id, + title: content.title.clone(), + cd_keys: content.cd_keys.clone(), + data: HashMap::new(), + extras: HashMap::new(), + }; + + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + + let key_root = content_root.join("keys"); + + if content.cd_keys.len() > 0 { + fs::create_dir_all(&key_root)?; + } + + for (key, value) in content.cd_keys { + let key_path = key_root.join(format!("{}.txt", key)); + let mut key_file = File::create(&key_path)?; + key_file.write_all(value.as_bytes())? + } + + for data in content.data { + let data_hash_saved = match content_info_saved.data.get(&content_id.to_string()) { + Some(value) => value.hash, + None => u64::min_value(), + }; + + let data_hash = models::get_hash(&data); + + if data_hash_saved == data_hash { + info!("{} already up to date.", &data.manual_url); + continue; + } + + let data_root = Path::new(&content_root).join(&data.language); + fs::create_dir_all(&data_root)?; + + let data_uri = format!("https://embed.gog.com{}", data.manual_url); + + info!("downloading {} for {}...", &data.manual_url, &content.title); + let filename = self.api_request_download(data_uri.as_str(), &data_root)?; + + let data_info = DataInfo { + hash: data_hash, + filename: filename, + language: data.language, + }; + + content_info + .data + .insert(data.manual_url.clone(), data_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + } + + for extra in content.extras { + let extra_hash_saved = match content_info_saved.extras.get(&content_id.to_string()) { + Some(value) => value.hash, + None => u64::min_value(), + }; + + let extra_hash = models::get_hash(&extra); + + if extra_hash_saved == extra_hash { + info!("{} already up to date.", &extra.manual_url); + continue; + } + + let extra_uri = format!("https://embed.gog.com{}", extra.manual_url); + + info!("downloading {} for {}...", &extra.name, &content.title); + let filename = self.api_request_download(extra_uri.as_str(), &content_root)?; + + let extra_info = ExtraInfo { + hash: extra_hash, + filename: filename, + name: extra.name, + }; + + content_info + .extras + .insert(extra.manual_url.clone(), extra_info); + ConfigFiles::save_to_path(&content_info_path, &content_info)?; + } + } + + Ok(()) +}*/