shu/gog-sync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch '4-support-movies' into 'master'

Browse Source 
Resolve "support movies"

Closes #4

See merge request !4
This commit is contained in:
Sebastian Hugentobler 2017-03-22 15:30:14 +00:00
commit 99b74d5abb
10 changed files with 349 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.editorconfig
View File

@ -7,3 +7,6 @@ trim_trailing_whitespace = true
insert_final_newline = true
indent_style = space
indent_size = 4
[*.md]
trim_trailing_whitespace = true

7
CHANGELOG.md
View File

@ -1,3 +1,10 @@
## 0.2.2 (2017-03-22)
- ability to filter movies by resolution
- ability to save movies seperately
- default value for storage config (fixes #6)
- ability to skip movie and game content
- fix a bug with truncating an http response
## 0.2.1 (2017-03-21)
- add instructions for getting the code if there is no token

2
Cargo.lock generated
View File

@ -1,6 +1,6 @@
[root]
name = "gog-sync"
version = "0.2.0"
version = "0.2.2"
dependencies = [
"chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)",

2
Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "gog-sync"
version = "0.2.1"
version = "0.2.2"
authors = ["Sebastian Hugentobler <sebastian@vanwa.ch>"]
description = "Synchronizes a GOG library with a local folder."
documentation = "https://docs.rs/crate/gog-sync"

76
README.md
View File

@ -1,4 +1,5 @@
# GOG-SYNC
## **Please note that this is alpha software, you should not trust it.**
A small tool to synchronize the stuff in a [GOG](https://www.gog.com/) library
with a local folder.
@ -7,8 +8,6 @@ It builds on the work of the [unofficial GOG API Documentation](https://gogapido
This is the first time I am building something with rust, so beware :)
**Please note that this is alpha software, you should not trust it.**
# Installation
Install from [crates.io](https://crates.io).
@ -26,13 +25,82 @@ For example on macOS or Linux
~/.config/gog-sync/config.json
```
It is in Json format and the only relevant key is `storage`. The rest is information
about content hashes.
A bare configuration with default values before first use:
```
{
"gameStorage": ".",
"movieStorage": ".",
"content": {},
"data": {},
"extras": {},
"osFilters": [],
"languageFilters": [],
"skipMovies": false,
"skipGames": false
}
```
- *gameStorage*: Where to save games
- *movieStorage*: Where to save movies
- *content*: A map, content id => hash
- *data*: A map, data url => hash
- *extras*: A map, extra url => hash
- *osFilters*: An array of operating systems. If it is not empty, game data is limited to the ones in the list.
- *languageFilters*: An array of languages. If it is not empty, game data is limited to the ones in the list.
- *resolutionFilters*: An array of resolutions. If it is not empty, movie data is limited to the ones in the list.
- *skipMovies*: Whether to skip movie content
- *skipGames*: Whether to skip game content
Valid values for *osFilter*:
- `linux`
- `mac`
- `windows`
An incomplete list of languages on gog:
- `english`
- `český`
- `deutsch`
- `español`
- `français`
- `italiano`
- `magyar`
- `polski`
- `русский`
- `中文`
An incomplete list of resolutions on gog:
- `DVD`
- `576p`
- `720p`
- `1080p`
- `4k`
You should have no need of changing `content`, `data` or `extras`, as these are
used to determine whether specific content is up to date.
# Usage
If you want to see the information log while running set `RUST_LOG=info`.
```
USAGE:
gog-sync [FLAGS] [OPTIONS]
FLAGS:
-h, --help Prints help information
-g, --skip-games Skip game content.
-f, --skip-movies Skip movie content.
-V, --version Prints version information
OPTIONS:
-s, --game-storage <FOLDER> Sets the download folder (defaults to the working directory).
-l, --language <FILTER> Only sync files for this comma seperated list of languages.
-m, --movie-storage <FOLDER> Sets the download folder for movies (defaults to the working directory).
-o, --os <FILTER> Only sync files for this comma seperated list of operating systems.
Valid values are 'linux', 'mac' and 'windows'.
-r, --resolution <FILTER> Only sync movies for this comma seperated list of resolutions.
```
---
```

10
src/configfiles.rs
View File

@ -7,6 +7,7 @@
use serde::{Deserialize, Serialize};
use serde_json;
use std::fmt;
use std::fs::File;
use std;
use std::io::{Read, Write};
@ -20,6 +21,15 @@ pub enum ConfigError {
SerdeJsonError(serde_json::Error),
}
impl fmt::Display for ConfigError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
ConfigError::IOError(ref err) => fmt::Display::fmt(err, f),
ConfigError::SerdeJsonError(ref err) => fmt::Display::fmt(err, f),
}
}
}
impl From<std::io::Error> for ConfigError {
fn from(e: std::io::Error) -> Self {
ConfigError::IOError(e)

228
src/gog.rs
View File

@ -13,10 +13,11 @@
use configfiles::{ConfigFiles, ConfigError};
use http::{Http, HttpError};
use models;
use models::{Token, Game, Installer, Config};
use models::{Token, Content, Data, Config};
use serde_json;
use serde_json::Value;
use std::collections::HashMap;
use std::fmt;
use std::fs;
use std::fs::File;
use std::io;
@ -33,6 +34,18 @@ pub enum GogError {
IOError(io::Error),
}
impl fmt::Display for GogError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
GogError::Error(ref err) => fmt::Display::fmt(err, f),
GogError::ConfigError(ref err) => fmt::Display::fmt(err, f),
GogError::HttpError(ref err) => fmt::Display::fmt(err, f),
GogError::SerdeError(ref err) => fmt::Display::fmt(err, f),
GogError::IOError(ref err) => fmt::Display::fmt(err, f),
}
}
}
impl From<ConfigError> for GogError {
fn from(e: ConfigError) -> Self {
GogError::ConfigError(e)
@ -109,8 +122,12 @@ impl<'a> Gog<'a> {
/// Uses a hash to figure out whether something has changed.
pub fn sync(&mut self,
storage_path: &str,
storage_path_movies: &str,
os_filters: &Vec<String>,
language_filters: &Vec<String>)
language_filters: &Vec<String>,
resolution_filters: &Vec<String>,
skip_movies: bool,
skip_games: bool)
-> Result<(), GogError> {
let configfiles = ConfigFiles::new();
let mut config: Config = match configfiles.load("config.json") {
@ -119,68 +136,90 @@ impl<'a> Gog<'a> {
error!("Configuration error, generating new one...");
Config {
storage: String::from(storage_path),
games: HashMap::new(),
installers: HashMap::new(),
game_storage: String::from(storage_path),
movie_storage: String::from(storage_path_movies),
content: HashMap::new(),
data: HashMap::new(),
extras: HashMap::new(),
os_filters: os_filters.clone(),
language_filters: language_filters.clone(),
resolution_filters: resolution_filters.clone(),
skip_movies: skip_movies,
skip_games: skip_games,
}
}
};
let game_ids = self.get_game_ids()?;
let content_ids = self.get_content_ids()?;
for game_id in game_ids {
let game_hash_saved = match config.games.get(&game_id.to_string()) {
for content_id in content_ids {
let content_hash_saved = match config.content.get(&content_id.to_string()) {
Some(value) => value.clone(),
None => u64::min_value(),
};
let game = self.get_game(game_id, &os_filters, &language_filters)?;
let game_hash = models::get_hash(&game);
let content = match self.get_content(content_id,
&os_filters,
&language_filters,
&resolution_filters) {
Ok(value) => value,
Err(error) => {
error!("{}: {}", &content_id, error);
continue;
}
};
if game_hash_saved == game_hash {
info!("{} already up to date.", &game.title);
if (content.is_movie && skip_movies) || (!content.is_movie && skip_games) {
info!("filtering {}", content.title);
continue;
}
let game_root = Path::new(storage_path).join(&game.title);
fs::create_dir_all(&game_root)?;
let content_hash = models::get_hash(&content);
if !game.cd_key.is_empty() {
let key_path = Path::new(game_root.as_os_str()).join("key.txt");
let mut key_file = File::create(&key_path)?;
key_file.write_all(game.cd_key.as_bytes())?
if content_hash_saved == content_hash {
info!("{} already up to date.", &content.title);
continue;
}
for installer in game.installers {
let installer_hash_saved = match config.installers.get(&installer.manual_url) {
let content_root = if content.is_movie {
Path::new(storage_path_movies).join(&content.title)
} else {
Path::new(storage_path).join(&content.title)
};
fs::create_dir_all(&content_root)?;
if !content.cd_key.is_empty() {
let key_path = Path::new(content_root.as_os_str()).join("key.txt");
let mut key_file = File::create(&key_path)?;
key_file.write_all(content.cd_key.as_bytes())?
}
for data in content.data {
let data_hash_saved = match config.data.get(&data.manual_url) {
Some(value) => value.clone(),
None => u64::min_value(),
};
let installer_hash = models::get_hash(&installer);
let data_hash = models::get_hash(&data);
if installer_hash_saved == installer_hash {
info!("{} already up to date.", &installer.manual_url);
if data_hash_saved == data_hash {
info!("{} already up to date.", &data.manual_url);
continue;
}
let installer_root = Path::new(&game_root).join(&installer.language);
fs::create_dir_all(&installer_root)?;
let data_root = Path::new(&content_root).join(&data.language);
fs::create_dir_all(&data_root)?;
let installer_uri = format!("https://embed.gog.com{}", installer.manual_url);
let data_uri = format!("https://embed.gog.com{}", data.manual_url);
info!("downloading {} for {}...",
&installer.manual_url,
&game.title);
self.http_client.download(installer_uri.as_str(), &installer_root)?;
info!("downloading {} for {}...", &data.manual_url, &content.title);
self.http_client.download(data_uri.as_str(), &data_root)?;
config.installers.insert(installer.manual_url, installer_hash);
config.data.insert(data.manual_url, data_hash);
configfiles.save("config.json", &config)?;
}
for extra in game.extras {
for extra in content.extras {
let extra_hash_saved = match config.extras.get(&extra.manual_url) {
Some(value) => value.clone(),
None => u64::min_value(),
@ -195,14 +234,14 @@ impl<'a> Gog<'a> {
let extra_uri = format!("https://embed.gog.com{}", extra.manual_url);
info!("downloading {} for {}...", &extra.name, &game.title);
self.http_client.download(extra_uri.as_str(), &game_root)?;
info!("downloading {} for {}...", &extra.name, &content.title);
self.http_client.download(extra_uri.as_str(), &content_root)?;
config.extras.insert(extra.manual_url, extra_hash);
configfiles.save("config.json", &config)?;
}
config.games.insert(game_id.to_string(), game_hash);
config.content.insert(content_id.to_string(), content_hash);
configfiles.save("config.json", &config)?;
}
@ -261,80 +300,97 @@ impl<'a> Gog<'a> {
}
}
fn get_game_ids(&mut self) -> Result<Vec<u64>, GogError> {
fn get_content_ids(&mut self) -> Result<Vec<u64>, GogError> {
let response = self.http_client.get(self.games_uri.as_str())?;
let game_ids_raw: Value = serde_json::from_str(response.as_str())?;
let game_ids_serde = &game_ids_raw["owned"];
let content_ids_raw: Value = serde_json::from_str(response.as_str())?;
let content_ids_serde = &content_ids_raw["owned"];
let mut game_ids: Vec<u64> = Vec::new();
let mut content_ids: Vec<u64> = Vec::new();
if !game_ids_serde.is_array() {
return Err(GogError::Error("Error parsing game ids."));
if !content_ids_serde.is_array() {
return Err(GogError::Error("Error parsing content ids."));
}
for game_id in game_ids_serde.as_array().unwrap() {
let game_id_parsed = game_id.as_u64().unwrap_or(0);
for content_id in content_ids_serde.as_array().unwrap() {
let content_id_parsed = content_id.as_u64().unwrap_or(0);
if game_id_parsed == 0 {
error!("Cant parse game id {}", game_id);
if content_id_parsed == 0 {
error!("Cant parse content id {}", content_id);
continue;
}
// The numbers in this list are excluded because they refer to
// favourites, promotions and such.
if [1, 2, 3, 4, 5].contains(&game_id_parsed) {
if [1, 2, 3, 4, 5].contains(&content_id_parsed) {
continue;
}
game_ids.push(game_id_parsed);
content_ids.push(content_id_parsed);
}
return Ok(game_ids);
return Ok(content_ids);
}
fn get_game(&mut self,
game_id: u64,
os_filters: &Vec<String>,
language_filters: &Vec<String>)
-> Result<Game, GogError> {
let game_uri = self.game_uri(game_id);
fn get_content(&mut self,
content_id: u64,
os_filters: &Vec<String>,
language_filters: &Vec<String>,
resolution_filters: &Vec<String>)
-> Result<Content, GogError> {
let content_uri = self.content_uri(content_id);
debug!("looking for information at {}...", &content_uri);
let response = self.http_client.get(game_uri.as_str())?;
let response = self.http_client.get(content_uri.as_str())?;
let mut game: Game = serde_json::from_str(&response)?;
let content_raw: Value = serde_json::from_str(response.as_str())?;
debug!("found {:?}", &content_raw);
let game_raw: Value = serde_json::from_str(response.as_str())?;
let downloads = &game_raw["downloads"];
let mut content: Content = serde_json::from_str(&response)?;
let downloads = &content_raw["downloads"];
if content_raw.is_object() && !content_raw.as_object().unwrap().contains_key("forumLink") {
return Err(GogError::Error("No forumLink property"));
}
let is_movie = match content_raw["forumLink"].as_str() {
Some(value) => value == "https://embed.gog.com/forum/movies",
None => false,
};
content.is_movie = is_movie;
debug!("processing installer fields: {:?}", &downloads);
for languages in downloads.as_array() {
for language in languages {
if !language.is_array() || language.as_array().unwrap().len() < 2 {
error!("Skipping a language for {}", game.title);
error!("Skipping a language for {}", content.title);
continue;
}
let installer_language = match language[0].as_str() {
let data_language = match language[0].as_str() {
Some(value) => value.to_lowercase(),
None => String::default(),
};
if installer_language.is_empty() {
error!("Skipping a language for {}", game.title);
if data_language.is_empty() {
error!("Skipping a language for {}", content.title);
continue;
}
if !language_filters.is_empty() && !language_filters.contains(&installer_language) {
info!("Skipping {} for {}", &installer_language, game.title);
if !is_movie && !language_filters.is_empty() &&
!language_filters.contains(&data_language) {
info!("Skipping {} for {}", &data_language, content.title);
continue;
}
for systems in language[1].as_object() {
for system in systems.keys() {
if !os_filters.is_empty() && !os_filters.contains(system) {
if is_movie && !os_filters.is_empty() && !os_filters.contains(system) {
info!("Skipping {} {} for {}",
&installer_language,
&data_language,
system,
game.title);
content.title);
continue;
}
@ -342,12 +398,32 @@ impl<'a> Gog<'a> {
for real_download in real_downloads.as_array() {
for download in real_download {
if !download.is_object() ||
!download.as_object().unwrap().contains_key("manualUrl") {
error!("Skipping an installer for {}", game.title);
!download.as_object().unwrap().contains_key("manualUrl") ||
!download.as_object().unwrap().contains_key("name") {
error!("Skipping data for {}", content.title);
continue;
}
let installer = Installer {
let name: &str = download["name"].as_str().unwrap();
if content.is_movie && !resolution_filters.is_empty() {
let mut found_resolution = false;
for resolution_filter in resolution_filters {
let filter = format!("({})", resolution_filter);
if name.ends_with(&filter) {
found_resolution = true;
break;
}
}
if !found_resolution {
info!("Skipping {}: not a suitable resolution.", name);
continue;
}
}
let data = Data {
manual_url: String::from(download["manualUrl"]
.as_str()
.unwrap()),
@ -355,10 +431,10 @@ impl<'a> Gog<'a> {
.as_str()
.unwrap_or("")),
os: system.clone(),
language: installer_language.clone(),
language: data_language.clone(),
};
game.installers.push(installer);
content.data.push(data);
}
}
}
@ -367,7 +443,7 @@ impl<'a> Gog<'a> {
}
}
Ok(game)
Ok(content)
}
fn auth_uri(&self, client_id: &str, redirect_uri: &str) -> String {
@ -403,8 +479,8 @@ impl<'a> Gog<'a> {
refresh_token = refresh_token)
}
fn game_uri(&self, game_id: u64) -> String {
format!("https://embed.gog.com/account/gameDetails/{game_id}.json",
game_id = game_id)
fn content_uri(&self, content_id: u64) -> String {
format!("https://embed.gog.com/account/gameDetails/{content_id}.json",
content_id = content_id)
}
}

27
src/http.rs
View File

@ -3,6 +3,7 @@
use curl;
use curl::easy::{Easy, List, WriteError};
use std::fmt;
use std::fs;
use std::fs::File;
use std::io;
@ -24,6 +25,18 @@ pub enum HttpError {
UrlParseError(url::ParseError),
}
impl fmt::Display for HttpError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
HttpError::Error(ref err) => fmt::Display::fmt(err, f),
HttpError::CurlError(ref err) => fmt::Display::fmt(err, f),
HttpError::Utf8Error(ref err) => fmt::Display::fmt(err, f),
HttpError::IOError(ref err) => fmt::Display::fmt(err, f),
HttpError::UrlParseError(ref err) => fmt::Display::fmt(err, f),
}
}
}
impl From<curl::Error> for HttpError {
fn from(e: curl::Error) -> Self {
HttpError::CurlError(e)
@ -79,20 +92,22 @@ impl Http {
/// http_client.get("https://discworld.com/");
/// ```
pub fn get(&mut self, uri: &str) -> Result<String, HttpError> {
let mut response_body = String::new();
let mut data = Vec::new();
self.curl.url(uri)?;
{
let mut transfer = self.curl.transfer();
transfer.write_function(|data| {
response_body = String::from(str::from_utf8(data).unwrap());
Ok(data.len())
transfer.write_function(|new_data| {
data.extend_from_slice(new_data);
Ok(new_data.len())
})?;
transfer.perform()?;
}
Ok(response_body)
match str::from_utf8(data.as_slice()) {
Ok(value) => Ok(value.to_owned()),
Err(error) => Err(HttpError::Utf8Error(error)),
}
}
/// Add a header to all future requests.

78
src/main.rs
View File

@ -1,19 +1,4 @@
//! Synchronizes a [GOG](https://www.gog.com/) library with a local folder.
//!
//! ```
//! USAGE:
//! gog-sync [OPTIONS]
//!
//! FLAGS:
//! -h, --help Prints help information
//! -V, --version Prints version information
//!
//! OPTIONS:
//! -l, --language <FILTER> Only sync files for this comma seperated list of languages.
//! -o, --os <FILTER> Only sync files for this comma seperated list of operating systems.
//! Valid values are 'linux', 'mac' and 'windows'.
//! -s, --storage <FOLDER> Sets the download folder (defaults to the working directory).
//! ```
extern crate chrono;
extern crate clap;
@ -43,15 +28,21 @@ fn main() {
env_logger::init().unwrap();
let matches = App::new("Gog Synchronizer")
.version("0.2.1")
.version("0.2.2")
.author("Sebastian Hugentobler <sebastian@vanwa.ch>")
.about("Synchronizes your gog library to a local folder.")
.arg(Arg::with_name("storage")
.arg(Arg::with_name("game-storage")
.short("s")
.long("storage")
.long("game-storage")
.value_name("FOLDER")
.help("Sets the download folder (defaults to the working directory).")
.takes_value(true))
.arg(Arg::with_name("movie-storage")
.short("m")
.long("movie-storage")
.value_name("FOLDER")
.help("Sets the download folder for movies (defaults to the working directory).")
.takes_value(true))
.arg(Arg::with_name("os")
.short("o")
.long("os")
@ -65,6 +56,22 @@ fn main() {
.value_name("FILTER")
.help("Only sync files for this comma seperated list of languages.")
.takes_value(true))
.arg(Arg::with_name("resolution")
.short("r")
.long("resolution")
.value_name("FILTER")
.help("Only sync movies for this comma seperated list of resolutions.")
.takes_value(true))
.arg(Arg::with_name("skip-movies")
.short("f")
.long("skip-movies")
.help("Skip movie content.")
.takes_value(false))
.arg(Arg::with_name("skip-games")
.short("g")
.long("skip-games")
.help("Skip game content.")
.takes_value(false))
.get_matches();
let configfiles = ConfigFiles::new();
@ -73,9 +80,14 @@ fn main() {
Err(_) => Config::new(),
};
let download_folder: &str = match matches.value_of("storage") {
let download_folder: &str = match matches.value_of("game-storage") {
Some(value) => value,
None => config.storage.as_str(),
None => config.game_storage.as_str(),
};
let download_folder_movies: &str = match matches.value_of("movie-storage") {
Some(value) => value,
None => config.movie_storage.as_str(),
};
let os_filters: Vec<String> = match matches.value_of("os") {
@ -88,8 +100,32 @@ fn main() {
None => config.language_filters,
};
let resolution_filters: Vec<String> = match matches.value_of("resolution") {
Some(value) => value.split(',').map(String::from).collect(),
None => config.resolution_filters,
};
let skip_movies = if matches.is_present("skip-movies") {
true
} else {
config.skip_movies
};
let skip_games = if matches.is_present("skip-games") {
true
} else {
config.skip_games
};
let mut http_client = Http::new();
let mut gog = Gog::new(&mut http_client);
gog.login().unwrap();
gog.sync(download_folder, &os_filters, &language_filters).unwrap();
gog.sync(download_folder,
download_folder_movies,
&os_filters,
&language_filters,
&resolution_filters,
skip_movies,
skip_games)
.unwrap();
}

36
src/models.rs
View File

@ -33,17 +33,25 @@ fn timestamp() -> i64 {
#[serde(rename_all = "camelCase")]
pub struct Config {
#[serde(default)]
pub storage: String,
pub game_storage: String,
#[serde(default)]
pub movie_storage: String,
#[serde(default = "default_map")]
pub games: HashMap<String, u64>,
pub content: HashMap<String, u64>,
#[serde(default = "default_map")]
pub installers: HashMap<String, u64>,
pub data: HashMap<String, u64>,
#[serde(default = "default_map")]
pub extras: HashMap<String, u64>,
#[serde(default = "default_list")]
pub os_filters: Vec<String>,
#[serde(default = "default_list")]
pub language_filters: Vec<String>,
#[serde(default = "default_list")]
pub resolution_filters: Vec<String>,
#[serde(default)]
pub skip_movies: bool,
#[serde(default)]
pub skip_games: bool,
}
fn default_map() -> HashMap<String, u64> {
@ -57,12 +65,16 @@ fn default_list() -> Vec<String> {
impl Config {
pub fn new() -> Config {
Config {
storage: String::from("."),
games: HashMap::new(),
installers: HashMap::new(),
game_storage: String::from("."),
movie_storage: String::from("."),
content: HashMap::new(),
data: HashMap::new(),
extras: HashMap::new(),
os_filters: Vec::new(),
language_filters: Vec::new(),
resolution_filters: Vec::new(),
skip_movies: false,
skip_games: false,
}
}
}
@ -70,15 +82,17 @@ impl Config {
#[derive(Hash)]
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Game {
pub struct Content {
pub title: String,
pub cd_key: String,
#[serde(skip_deserializing)]
pub installers: Vec<Installer>,
pub is_movie: bool,
#[serde(skip_deserializing)]
pub data: Vec<Data>,
pub extras: Vec<Extra>,
}
impl fmt::Display for Game {
impl fmt::Display for Content {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "({})", self.title)
}
@ -87,14 +101,14 @@ impl fmt::Display for Game {
#[derive(Hash)]
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Installer {
pub struct Data {
pub manual_url: String,
pub version: String,
pub os: String,
pub language: String,
}
impl fmt::Display for Installer {
impl fmt::Display for Data {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f,
"({}, {}, {}, {})",