move content deserialization in module file

This commit is contained in:
Sebastian Hugentobler 2017-05-10 12:03:48 +02:00
parent 2f85d2459f
commit 865f398013
2 changed files with 168 additions and 154 deletions

View File

@ -17,12 +17,10 @@ use models::content::Content;
use models::contentinfo::ContentInfo; use models::contentinfo::ContentInfo;
use models::datainfo::DataInfo; use models::datainfo::DataInfo;
use models::extrainfo::ExtraInfo; use models::extrainfo::ExtraInfo;
use models::data::Data;
use models::token::Token; use models::token::Token;
use serde_json; use serde_json;
use serde_json::Value; use serde_json::Value;
use std::collections::HashMap; use std::collections::HashMap;
use std::collections::BTreeMap;
use std::fmt; use std::fmt;
use std::fs; use std::fs;
use std::fs::File; use std::fs::File;
@ -352,44 +350,6 @@ impl<'a> Gog<'a> {
return Ok(content_ids); return Ok(content_ids);
} }
fn parse_cd_keys(&self, content_title: &str, raw_cd_keys: &str) -> BTreeMap<String, String> {
let mut cd_keys = BTreeMap::new();
let mut raw_cd_keys_fix = raw_cd_keys.to_owned();
if raw_cd_keys_fix.contains("<span>") {
raw_cd_keys_fix = raw_cd_keys_fix
.replace("</span><span>", ":")
.replace("<span>", "")
.replace("</span>", "");
}
raw_cd_keys_fix = raw_cd_keys_fix.replace("<br>", ":").replace("::", ":");
if raw_cd_keys_fix.contains(":") {
let splitted_keys = raw_cd_keys_fix.split(":");
let mut key_names: Vec<String> = Vec::new();
let mut key_values: Vec<String> = Vec::new();
for (token_index, token) in splitted_keys.enumerate() {
if token_index % 2 == 0 {
key_names.push(token.to_owned());
} else {
key_values.push(token.trim().to_owned());
}
}
for (index, key_name) in key_names.iter().enumerate() {
let key_value = key_values[index].clone();
cd_keys.insert(key_name.clone(), key_value);
}
} else if !raw_cd_keys_fix.is_empty() {
cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned());
}
return cd_keys;
}
fn api_request_ensure_token(&mut self, response: &str) -> Result<bool, GogError> { fn api_request_ensure_token(&mut self, response: &str) -> Result<bool, GogError> {
let response_json: Value = match serde_json::from_str(response) { let response_json: Value = match serde_json::from_str(response) {
Ok(value) => value, Ok(value) => value,
@ -436,118 +396,11 @@ impl<'a> Gog<'a> {
let response = self.api_request_get(content_uri.as_str())?; let response = self.api_request_get(content_uri.as_str())?;
let content_raw: Value = serde_json::from_str(response.as_str())?; models::content::deserialize(content_id,
debug!("found {:?}", &content_raw); response.as_str(),
os_filters,
let mut content: Content = serde_json::from_str(&response)?; language_filters,
content.id = content_id; resolution_filters)
let downloads = &content_raw["downloads"];
if content_raw.is_object() &&
!content_raw
.as_object()
.unwrap()
.contains_key("forumLink") {
return Err(GogError::Error("No forumLink property"));
}
let is_movie = match content_raw["forumLink"].as_str() {
Some(value) => value == "https://embed.gog.com/forum/movies",
None => false,
};
content.is_movie = is_movie;
if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") {
let cd_keys_raw = content_raw["cdKey"].as_str().unwrap();
content.cd_keys = self.parse_cd_keys(&content.title, cd_keys_raw);
}
debug!("processing installer fields: {:?}", &downloads);
for languages in downloads.as_array() {
for language in languages {
if !language.is_array() || language.as_array().unwrap().len() < 2 {
error!("Skipping a language for {}", content.title);
continue;
}
let data_language = match language[0].as_str() {
Some(value) => value.to_lowercase(),
None => String::default(),
};
if data_language.is_empty() {
error!("Skipping a language for {}", content.title);
continue;
}
if !is_movie && !language_filters.is_empty() &&
!language_filters.contains(&data_language) {
info!("Skipping {} for {}", &data_language, content.title);
continue;
}
for systems in language[1].as_object() {
for system in systems.keys() {
if is_movie && !os_filters.is_empty() && !os_filters.contains(system) {
info!("Skipping {} {} for {}",
&data_language,
system,
content.title);
continue;
}
for real_downloads in systems.get(system) {
for real_download in real_downloads.as_array() {
for download in real_download {
if !download.is_object() ||
!download.as_object().unwrap().contains_key("manualUrl") ||
!download.as_object().unwrap().contains_key("name") {
error!("Skipping data for {}", content.title);
continue;
}
let name: &str = download["name"].as_str().unwrap();
if content.is_movie && !resolution_filters.is_empty() {
let mut found_resolution = false;
for resolution_filter in resolution_filters {
let filter = format!("({})", resolution_filter);
if name.ends_with(&filter) {
found_resolution = true;
break;
}
}
if !found_resolution {
info!("Skipping {}: not a suitable resolution.", name);
continue;
}
}
let data = Data {
manual_url: String::from(download["manualUrl"]
.as_str()
.unwrap()),
version: String::from(download["version"]
.as_str()
.unwrap_or("")),
os: system.clone(),
language: data_language.clone(),
};
content.data.push(data);
}
}
}
}
}
}
}
Ok(content)
} }
fn games_uri(&self) -> String { fn games_uri(&self) -> String {

View File

@ -1,7 +1,10 @@
use gog::GogError;
use models::data::Data; use models::data::Data;
use models::extra::Extra; use models::extra::Extra;
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Deserializer}; use serde::{Deserialize, Deserializer};
use serde_json;
use serde_json::Value;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::fmt; use std::fmt;
@ -11,7 +14,7 @@ use std::fmt;
pub struct Content { pub struct Content {
#[serde(skip_deserializing)] #[serde(skip_deserializing)]
pub id: u64, pub id: u64,
#[serde(deserialize_with = "normalize_title")] #[serde(deserialize_with = "deserialize_title")]
pub title: String, pub title: String,
#[serde(skip_deserializing)] #[serde(skip_deserializing)]
#[serde(rename(deserialize = "cdKey"))] #[serde(rename(deserialize = "cdKey"))]
@ -21,6 +24,7 @@ pub struct Content {
#[serde(skip_deserializing)] #[serde(skip_deserializing)]
pub data: Vec<Data>, pub data: Vec<Data>,
pub extras: Vec<Extra>, pub extras: Vec<Extra>,
pub dlcs: Vec<Content>,
} }
impl fmt::Display for Content { impl fmt::Display for Content {
@ -29,7 +33,7 @@ impl fmt::Display for Content {
} }
} }
fn normalize_title<D>(deserializer: D) -> Result<String, D::Error> fn deserialize_title<D>(deserializer: D) -> Result<String, D::Error>
where D: Deserializer where D: Deserializer
{ {
let raw_title = String::deserialize(deserializer)?.replace(":", " - "); let raw_title = String::deserialize(deserializer)?.replace(":", " - ");
@ -47,3 +51,160 @@ fn normalize_title<D>(deserializer: D) -> Result<String, D::Error>
Ok(title_whitespace) Ok(title_whitespace)
} }
fn deserialize_cd_keys(content_title: &str, raw_cd_keys: &str) -> BTreeMap<String, String> {
let mut cd_keys = BTreeMap::new();
let mut raw_cd_keys_fix = raw_cd_keys.to_owned();
if raw_cd_keys_fix.contains("<span>") {
raw_cd_keys_fix = raw_cd_keys_fix
.replace("</span><span>", ":")
.replace("<span>", "")
.replace("</span>", "");
}
raw_cd_keys_fix = raw_cd_keys_fix.replace("<br>", ":").replace("::", ":");
if raw_cd_keys_fix.contains(":") {
let splitted_keys = raw_cd_keys_fix.split(":");
let mut key_names: Vec<String> = Vec::new();
let mut key_values: Vec<String> = Vec::new();
for (token_index, token) in splitted_keys.enumerate() {
if token_index % 2 == 0 {
key_names.push(token.to_owned());
} else {
key_values.push(token.trim().to_owned());
}
}
for (index, key_name) in key_names.iter().enumerate() {
let key_value = key_values[index].clone();
cd_keys.insert(key_name.clone(), key_value);
}
} else if !raw_cd_keys_fix.is_empty() {
cd_keys.insert(content_title.to_owned(), raw_cd_keys_fix.trim().to_owned());
}
cd_keys
}
pub fn deserialize(content_id: u64,
content_string: &str,
os_filters: &Vec<String>,
language_filters: &Vec<String>,
resolution_filters: &Vec<String>)
-> Result<Content, GogError> {
let content_raw: Value = serde_json::from_str(content_string)?;
debug!("found {:?}", &content_raw);
let mut content: Content = serde_json::from_str(content_string)?;
content.id = content_id;
let downloads = &content_raw["downloads"];
if content_raw.is_object() &&
!content_raw
.as_object()
.unwrap()
.contains_key("forumLink") {
return Err(GogError::Error("No forumLink property"));
}
let is_movie = match content_raw["forumLink"].as_str() {
Some(value) => value == "https://embed.gog.com/forum/movies",
None => false,
};
content.is_movie = is_movie;
if content_raw.is_object() && content_raw.as_object().unwrap().contains_key("cdKey") {
let cd_keys_raw = content_raw["cdKey"].as_str().unwrap();
content.cd_keys = deserialize_cd_keys(&content.title, cd_keys_raw);
}
debug!("processing installer fields: {:?}", &downloads);
for languages in downloads.as_array() {
for language in languages {
if !language.is_array() || language.as_array().unwrap().len() < 2 {
error!("Skipping a language for {}", content.title);
continue;
}
let data_language = match language[0].as_str() {
Some(value) => value.to_lowercase(),
None => String::default(),
};
if data_language.is_empty() {
error!("Skipping a language for {}", content.title);
continue;
}
if !is_movie && !language_filters.is_empty() &&
!language_filters.contains(&data_language) {
info!("Skipping {} for {}", &data_language, content.title);
continue;
}
for systems in language[1].as_object() {
for system in systems.keys() {
if is_movie && !os_filters.is_empty() && !os_filters.contains(system) {
info!("Skipping {} {} for {}",
&data_language,
system,
content.title);
continue;
}
for real_downloads in systems.get(system) {
for real_download in real_downloads.as_array() {
for download in real_download {
if !download.is_object() ||
!download.as_object().unwrap().contains_key("manualUrl") ||
!download.as_object().unwrap().contains_key("name") {
error!("Skipping data for {}", content.title);
continue;
}
let name: &str = download["name"].as_str().unwrap();
if content.is_movie && !resolution_filters.is_empty() {
let mut found_resolution = false;
for resolution_filter in resolution_filters {
let filter = format!("({})", resolution_filter);
if name.ends_with(&filter) {
found_resolution = true;
break;
}
}
if !found_resolution {
info!("Skipping {}: not a suitable resolution.", name);
continue;
}
}
let data = Data {
manual_url: String::from(download["manualUrl"]
.as_str()
.unwrap()),
version:
String::from(download["version"].as_str().unwrap_or("")),
os: system.clone(),
language: data_language.clone(),
};
content.data.push(data);
}
}
}
}
}
}
}
Ok(content)
}