save dlcs

This commit is contained in:
Sebastian Hugentobler 2017-05-10 14:19:09 +02:00
parent 865f398013
commit 7a40242d67
3 changed files with 461 additions and 89 deletions

View File

@ -15,15 +15,18 @@ use http::{Http, HttpError};
use models; use models;
use models::content::Content; use models::content::Content;
use models::contentinfo::ContentInfo; use models::contentinfo::ContentInfo;
use models::data::Data;
use models::datainfo::DataInfo; use models::datainfo::DataInfo;
use models::extra::Extra;
use models::extrainfo::ExtraInfo; use models::extrainfo::ExtraInfo;
use models::token::Token; use models::token::Token;
use serde_json; use serde_json;
use serde_json::Value; use serde_json::Value;
use std::collections::BTreeMap;
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt;
use std::fs; use std::fs;
use std::fs::File; use std::fs::File;
use std::fmt;
use std::io; use std::io;
use std::io::Write; use std::io::Write;
use std::path::Path; use std::path::Path;
@ -131,7 +134,7 @@ impl<'a> Gog<'a> {
/// Syncs the contents of a gog account with a local folder. /// Syncs the contents of a gog account with a local folder.
/// Uses a hash to figure out whether something has changed. /// Uses a hash to figure out whether something has changed.
pub fn sync(&mut self, pub fn sync(&mut self,
storage_path: &str, storage_path_games: &str,
storage_path_movies: &str, storage_path_movies: &str,
os_filters: &Vec<String>, os_filters: &Vec<String>,
language_filters: &Vec<String>, language_filters: &Vec<String>,
@ -153,120 +156,192 @@ impl<'a> Gog<'a> {
} }
}; };
let content_root = if content.is_movie {
Path::new(storage_path_movies).join(&content.title)
} else {
Path::new(storage_path).join(&content.title)
};
let content_info_path = Path::new(&content_root).join("info.json");
let content_info_saved =
match ConfigFiles::load_from_path::<ContentInfo>(&content_info_path) {
Ok(value) => value,
Err(_) => ContentInfo::new(),
};
if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) { if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) {
info!("filtering {}", content.title); info!("filtering {}", content.title);
continue; continue;
} }
let content_hash = models::get_hash(&content); match self.sync_content(content, storage_path_games, storage_path_movies) {
Ok(_) => (),
if content_info_saved.hash == content_hash { Err(error) => {
info!("{} already up to date.", &content.title); warn!("{:?}", error);
continue; continue;
}
} }
}
Ok(())
}
fs::create_dir_all(&content_root)?; fn sync_content(&mut self,
content: Content,
storage_path_games: &str,
storage_path_movies: &str)
-> Result<(), GogError> {
let mut content_info = ContentInfo { let content_root = if content.is_movie {
hash: content_hash, Path::new(storage_path_movies).join(&content.title)
id: content_id, } else {
title: content.title.clone(), Path::new(storage_path_games).join(&content.title)
cd_keys: content.cd_keys.clone(), };
data: HashMap::new(),
extras: HashMap::new(), let content_info_path = Path::new(&content_root).join("info.json");
let content_info_saved =
match ConfigFiles::load_from_path::<ContentInfo>(&content_info_path) {
Ok(value) => value,
Err(_) => ContentInfo::new(),
}; };
ConfigFiles::save_to_path(&content_info_path, &content_info)?; let content_hash = models::get_hash(&content);
let key_root = content_root.join("keys"); if content_info_saved.hash == content_hash {
info!("{} already up to date.", &content.title);
return Ok(());
}
if content.cd_keys.len() > 0 { fs::create_dir_all(&content_root)?;
fs::create_dir_all(&key_root)?;
}
for (key, value) in content.cd_keys { let mut content_info = ContentInfo {
let key_path = key_root.join(format!("{}.txt", key)); hash: content_hash,
let mut key_file = File::create(&key_path)?; id: content.id,
key_file.write_all(value.as_bytes())? title: content.title.clone(),
} cd_keys: content.cd_keys.clone(),
data: HashMap::new(),
extras: HashMap::new(),
};
for data in content.data { ConfigFiles::save_to_path(&content_info_path, &content_info)?;
let data_hash_saved = match content_info_saved.data.get(&content_id.to_string()) {
Some(value) => value.hash,
None => u64::min_value(),
};
let data_hash = models::get_hash(&data); self.save_keys(content.cd_keys, &content_root)?;
if data_hash_saved == data_hash { for data in content.data {
info!("{} already up to date.", &data.manual_url); self.save_data(content.title.as_str(),
continue; data,
} &content_root,
&content_info_path,
&content_info_saved,
&mut content_info)?;
}
let data_root = Path::new(&content_root).join(&data.language); let dlc_root = Path::new(&storage_path_games)
fs::create_dir_all(&data_root)?; .join(&content.title)
.join("dlcs");
let data_uri = format!("https://embed.gog.com{}", data.manual_url); for dlc in content.dlcs {
self.sync_content(dlc,
dlc_root.to_string_lossy().as_ref(),
storage_path_movies)?;
}
info!("downloading {} for {}...", &data.manual_url, &content.title); for extra in content.extras {
let filename = self.api_request_download(data_uri.as_str(), &data_root)?; self.save_extra(content.title.as_str(),
extra,
&content_root,
&content_info_path,
&content_info_saved,
&mut content_info)?;
}
let data_info = DataInfo { Ok(())
hash: data_hash, }
filename: filename,
language: data.language,
};
content_info fn save_file(&mut self,
.data content_title: &str,
.insert(data.manual_url.clone(), data_info); relative_uri: &str,
ConfigFiles::save_to_path(&content_info_path, &content_info)?; data_root: &PathBuf)
} -> Result<String, GogError> {
fs::create_dir_all(&data_root)?;
let data_uri = format!("https://embed.gog.com{}", relative_uri);
for extra in content.extras { info!("downloading {} for {}...", &relative_uri, content_title);
let extra_hash_saved = self.api_request_download(data_uri.as_str(), &data_root)
match content_info_saved.extras.get(&content_id.to_string()) { }
Some(value) => value.hash,
None => u64::min_value(),
};
let extra_hash = models::get_hash(&extra); fn save_data(&mut self,
content_title: &str,
data: Data,
content_root: &PathBuf,
content_info_path: &PathBuf,
content_info_saved: &ContentInfo,
content_info: &mut ContentInfo)
-> Result<(), GogError> {
let hash_saved = match content_info_saved.data.get(data.manual_url.as_str()) {
Some(value) => value.hash,
None => u64::min_value(),
};
if extra_hash_saved == extra_hash { let hash = models::get_hash(&data);
info!("{} already up to date.", &extra.manual_url); if hash_saved == hash {
continue; info!("{} already up to date.", &data.manual_url);
} return Ok(());
}
let extra_uri = format!("https://embed.gog.com{}", extra.manual_url); let data_root = Path::new(&content_root).join(&data.language);
info!("downloading {} for {}...", &extra.name, &content.title); let filename = self.save_file(content_title, &data.manual_url, &data_root)?;
let filename = self.api_request_download(extra_uri.as_str(), &content_root)?;
let extra_info = ExtraInfo { let data_info = DataInfo {
hash: extra_hash, hash: hash,
filename: filename, filename: filename,
name: extra.name, language: data.language,
}; };
content_info content_info
.extras .data
.insert(extra.manual_url.clone(), extra_info); .insert(data.manual_url.clone(), data_info);
ConfigFiles::save_to_path(&content_info_path, &content_info)?; ConfigFiles::save_to_path(&content_info_path, &content_info)?;
}
Ok(())
}
fn save_extra(&mut self,
content_title: &str,
extra: Extra,
content_root: &PathBuf,
content_info_path: &PathBuf,
content_info_saved: &ContentInfo,
content_info: &mut ContentInfo)
-> Result<(), GogError> {
let hash_saved = match content_info_saved.data.get(extra.manual_url.as_str()) {
Some(value) => value.hash,
None => u64::min_value(),
};
let hash = models::get_hash(&extra);
if hash_saved == hash {
info!("{} already up to date.", &extra.manual_url);
return Ok(());
}
let filename = self.save_file(content_title, &extra.manual_url, &content_root)?;
let extra_info = ExtraInfo {
hash: hash,
filename: filename,
name: extra.name,
};
content_info
.extras
.insert(extra.manual_url.clone(), extra_info);
ConfigFiles::save_to_path(&content_info_path, &content_info)?;
Ok(())
}
fn save_keys(&mut self,
cd_keys: BTreeMap<String, String>,
content_root: &PathBuf)
-> Result<(), GogError> {
let key_root = content_root.join("keys");
if cd_keys.len() > 0 {
fs::create_dir_all(&key_root)?;
}
for (key, value) in cd_keys {
let key_path = key_root.join(format!("{}.txt", key));
let mut key_file = File::create(&key_path)?;
key_file.write_all(value.as_bytes())?
} }
Ok(()) Ok(())
@ -347,7 +422,8 @@ impl<'a> Gog<'a> {
content_ids.push(content_id_parsed); content_ids.push(content_id_parsed);
} }
return Ok(content_ids); //Ok(content_ids)
Ok(vec![1207658995])
} }
fn api_request_ensure_token(&mut self, response: &str) -> Result<bool, GogError> { fn api_request_ensure_token(&mut self, response: &str) -> Result<bool, GogError> {

View File

@ -102,6 +102,7 @@ pub fn deserialize(content_id: u64,
let mut content: Content = serde_json::from_str(content_string)?; let mut content: Content = serde_json::from_str(content_string)?;
content.id = content_id; content.id = content_id;
let dlcs = &content_raw["dlcs"];
let downloads = &content_raw["downloads"]; let downloads = &content_raw["downloads"];
if content_raw.is_object() && if content_raw.is_object() &&
@ -124,6 +125,24 @@ pub fn deserialize(content_id: u64,
content.cd_keys = deserialize_cd_keys(&content.title, cd_keys_raw); content.cd_keys = deserialize_cd_keys(&content.title, cd_keys_raw);
} }
match dlcs.as_array() {
Some(value) => {
if value.len() > 0 {
debug!("processing dlc fields: {:?}", &value);
for dlc in value {
let dlc = deserialize(content_id,
serde_json::to_string(&dlc).unwrap().as_str(),
os_filters,
language_filters,
resolution_filters)?;
content.dlcs.push(dlc);
}
}
}
None => (),
}
debug!("processing installer fields: {:?}", &downloads); debug!("processing installer fields: {:?}", &downloads);
for languages in downloads.as_array() { for languages in downloads.as_array() {
for language in languages { for language in languages {

277
src/sync.rs Normal file
View File

@ -0,0 +1,277 @@
use configfiles::ConfigFiles;
use gog::{Gog, GogError};
use http::{Http, HttpError};
use models;
use models::content::Content;
use models::contentinfo::ContentInfo;
use models::data::Data;
use models::datainfo::DataInfo;
use models::extra::Extra;
use std::collections::HashMap;
use std::collections::BTreeMap;
use std::io;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::fs;
use std::fs::File;
pub fn sync_content(content: Content,
storage_path_games: &str,
storage_path_movies: &str,
os_filters: &Vec<String>,
language_filters: &Vec<String>,
resolution_filters: &Vec<String>,
skip_movies: bool,
skip_games: bool)
-> Result<(), GogError> {
let content_root = if content.is_movie {
Path::new(storage_path_movies).join(&content.title)
} else {
Path::new(storage_path_games).join(&content.title)
};
let content_info_path = Path::new(&content_root).join("info.json");
let content_info_saved = match ConfigFiles::load_from_path::<ContentInfo>(&content_info_path) {
Ok(value) => value,
Err(_) => ContentInfo::new(),
};
if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) {
info!("filtering {}", content.title);
return Ok(());
}
let content_hash = models::get_hash(&content);
if content_info_saved.hash == content_hash {
info!("{} already up to date.", &content.title);
return Ok(());
}
fs::create_dir_all(&content_root)?;
let mut content_info = ContentInfo {
hash: content_hash,
id: content.id,
title: content.title.clone(),
cd_keys: content.cd_keys.clone(),
data: HashMap::new(),
extras: HashMap::new(),
};
ConfigFiles::save_to_path(&content_info_path, &content_info)?;
save_keys(content.cd_keys, &content_root)?;
for data in content.data {
save_data(content.title.as_str(),
data,
&content_root,
&content_info_path,
&content_info_saved,
&mut content_info);
}
Ok(())
}
fn save_data(content_title: &str,
data: Data,
content_root: &PathBuf,
content_info_path: &PathBuf,
content_info_saved: &ContentInfo,
content_info: &mut ContentInfo)
-> Result<(), GogError> {
let data_hash_saved = match content_info_saved.data.get(data.manual_url.as_str()) {
Some(value) => value.hash,
None => u64::min_value(),
};
let data_hash = models::get_hash(&data);
if data_hash_saved == data_hash {
info!("{} already up to date.", &data.manual_url);
return Ok(());
}
let data_root = Path::new(&content_root).join(&data.language);
fs::create_dir_all(&data_root)?;
let data_uri = format!("https://embed.gog.com{}", data.manual_url);
info!("downloading {} for {}...", &data.manual_url, content_title);
let filename = self.api_request_download(data_uri.as_str(), &data_root)?;
let data_info = DataInfo {
hash: data_hash,
filename: filename,
language: data.language,
};
content_info
.data
.insert(data.manual_url.clone(), data_info);
ConfigFiles::save_to_path(&content_info_path, &content_info)?;
Ok(())
}
fn save_keys(cd_keys: BTreeMap<String, String>, content_root: &PathBuf) -> Result<(), GogError> {
let key_root = content_root.join("keys");
if cd_keys.len() > 0 {
fs::create_dir_all(&key_root)?;
}
for (key, value) in cd_keys {
let key_path = key_root.join(format!("{}.txt", key));
let mut key_file = File::create(&key_path)?;
key_file.write_all(value.as_bytes())?
}
Ok(())
}
/*
/// Syncs the contents of a gog account with a local folder.
/// Uses a hash to figure out whether something has changed.
pub fn syncold(storage_path: &str,
storage_path_movies: &str,
os_filters: &Vec<String>,
language_filters: &Vec<String>,
resolution_filters: &Vec<String>,
skip_movies: bool,
skip_games: bool)
-> Result<(), GogError> {
let content_ids = self.get_content_ids()?;
for content_id in content_ids {
let content = match self.get_content(content_id,
&os_filters,
&language_filters,
&resolution_filters) {
Ok(value) => value,
Err(error) => {
error!("{}: {}", &content_id, error);
continue;
}
};
let content_root = if content.is_movie {
Path::new(storage_path_movies).join(&content.title)
} else {
Path::new(storage_path).join(&content.title)
};
let content_info_path = Path::new(&content_root).join("info.json");
let content_info_saved =
match ConfigFiles::load_from_path::<ContentInfo>(&content_info_path) {
Ok(value) => value,
Err(_) => ContentInfo::new(),
};
if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) {
info!("filtering {}", content.title);
continue;
}
let content_hash = models::get_hash(&content);
if content_info_saved.hash == content_hash {
info!("{} already up to date.", &content.title);
continue;
}
fs::create_dir_all(&content_root)?;
let mut content_info = ContentInfo {
hash: content_hash,
id: content_id,
title: content.title.clone(),
cd_keys: content.cd_keys.clone(),
data: HashMap::new(),
extras: HashMap::new(),
};
ConfigFiles::save_to_path(&content_info_path, &content_info)?;
let key_root = content_root.join("keys");
if content.cd_keys.len() > 0 {
fs::create_dir_all(&key_root)?;
}
for (key, value) in content.cd_keys {
let key_path = key_root.join(format!("{}.txt", key));
let mut key_file = File::create(&key_path)?;
key_file.write_all(value.as_bytes())?
}
for data in content.data {
let data_hash_saved = match content_info_saved.data.get(&content_id.to_string()) {
Some(value) => value.hash,
None => u64::min_value(),
};
let data_hash = models::get_hash(&data);
if data_hash_saved == data_hash {
info!("{} already up to date.", &data.manual_url);
continue;
}
let data_root = Path::new(&content_root).join(&data.language);
fs::create_dir_all(&data_root)?;
let data_uri = format!("https://embed.gog.com{}", data.manual_url);
info!("downloading {} for {}...", &data.manual_url, &content.title);
let filename = self.api_request_download(data_uri.as_str(), &data_root)?;
let data_info = DataInfo {
hash: data_hash,
filename: filename,
language: data.language,
};
content_info
.data
.insert(data.manual_url.clone(), data_info);
ConfigFiles::save_to_path(&content_info_path, &content_info)?;
}
for extra in content.extras {
let extra_hash_saved = match content_info_saved.extras.get(&content_id.to_string()) {
Some(value) => value.hash,
None => u64::min_value(),
};
let extra_hash = models::get_hash(&extra);
if extra_hash_saved == extra_hash {
info!("{} already up to date.", &extra.manual_url);
continue;
}
let extra_uri = format!("https://embed.gog.com{}", extra.manual_url);
info!("downloading {} for {}...", &extra.name, &content.title);
let filename = self.api_request_download(extra_uri.as_str(), &content_root)?;
let extra_info = ExtraInfo {
hash: extra_hash,
filename: filename,
name: extra.name,
};
content_info
.extras
.insert(extra.manual_url.clone(), extra_info);
ConfigFiles::save_to_path(&content_info_path, &content_info)?;
}
}
Ok(())
}*/