shu/ecload
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change from rust to go as I could not figure out the image loading problem in rust

Browse Source
This commit is contained in:
Sebastian Hugentobler 2019-05-27 16:52:48 +02:00
parent eb7502f52e
commit e3b2b03d55
23 changed files with 631 additions and 2680 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.gitignore vendored
View File

@ -1,5 +1,12 @@
*~
.DS_Store
*.swp
/target
**/*.rs.bk
**/.idea/workspace.xml
**/.idea/tasks.xml
*.exe
*.dll
*.so
*.dylib
*.test
*.out
.glide/

6
.idea/copyright/MPL_2_0.xml Normal file
View File

@ -0,0 +1,6 @@
<component name="CopyrightManager">
<copyright>
<option name="notice" value="/* This Source Code Form is subject to the terms of the Mozilla Public&#10; * License, v. 2.0. If a copy of the MPL was not distributed with this&#10; * file, You can obtain one at https://mozilla.org/MPL/2.0/. */" />
<option name="myName" value="MPL-2.0" />
</copyright>
</component>

3
.idea/copyright/profiles_settings.xml Normal file
View File

@ -0,0 +1,3 @@
<component name="CopyrightManager">
<settings default="MPL-2.0" />
</component>

13
.idea/misc.xml
View File

@ -1,19 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CargoProjects">
<cargoProject FILE="$PROJECT_DIR$/Cargo.toml" />
</component>
<component name="ProjectRootManager">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="RustProjectSettings">
<option name="toolchainHomeDirectory" value="$USER_HOME$/.cargo/bin" />
</component>
<component name="SbtLocalSettings">
<option name="projectSyncType">
<map>
<entry key="$PROJECT_DIR$/../../dev/gitsourcerepositories/dev-ding-10" value="PREVIEW" />
</map>
</option>
</component>
</project>

6
.idea/vcs.xml
View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

436
.idea/workspace.xml
View File

@ -1,51 +1,51 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="a321fd91-c97c-4f17-b620-fd9934708617" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/src/error.rs" beforeDir="false" afterPath="$PROJECT_DIR$/src/error.rs" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/loader.rs" beforeDir="false" afterPath="$PROJECT_DIR$/src/loader.rs" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/main.rs" beforeDir="false" afterPath="$PROJECT_DIR$/src/main.rs" afterDir="false" />
</list>
<list default="true" id="5f223132-21fb-4b87-a6ba-b086fa9127d3" name="Default Changelist" comment="" />
<ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="DefaultGradleProjectSettings">
<option name="testRunner" value="GRADLE" />
<option name="delegatedBuild" value="true" />
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main.rs">
<leaf>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/pkg/ecload/ecload.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="355">
<caret line="64" column="23" lean-forward="true" selection-start-line="64" selection-start-column="23" selection-end-line="64" selection-end-column="23" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/loader.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="249">
<caret line="222" column="47" selection-start-line="222" selection-start-column="47" selection-end-line="222" selection-end-column="47" />
<state relative-caret-position="217">
<caret line="31" column="19" lean-forward="true" selection-start-line="31" selection-start-column="19" selection-end-line="31" selection-end-column="19" />
<folding>
<element signature="e#4669#4670#0" expanded="true" />
<element signature="e#4723#4724#0" expanded="true" />
<element signature="e#223#321#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/error.rs">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pkg/ecload/fetcher.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="16" column="1" selection-start-line="16" selection-start-column="1" selection-end-line="16" selection-end-column="1" />
<state relative-caret-position="-14">
<caret line="17" column="43" selection-start-line="17" selection-start-column="43" selection-end-line="17" selection-end-column="43" />
<folding>
<element signature="e#353#354#0" expanded="true" />
<element signature="e#383#384#0" expanded="true" />
<element signature="e#223#314#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pkg/ecload/pdf.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="61">
<caret line="5" lean-forward="true" selection-start-line="5" selection-end-line="5" />
<folding>
<element signature="e#223#291#0" expanded="true" />
</folding>
</state>
</provider>
@ -56,49 +56,51 @@
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Rust File" />
<option value="Go File" />
</list>
</option>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>got</find>
<find>dpi</find>
<find>Px</find>
<find>PageFormat</find>
<find>format</find>
<find>saved</find>
<find>error!</find>
<find>exists</find>
<find>RegisterImageOptionsReader</find>
<find>downloadToFile</find>
</findStrings>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
<component name="GOROOT" path="$PROJECT_DIR$/../go/go1.12.5" />
<component name="GoLibraries">
<option name="indexEntireGoPath" value="false" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/printpdf-0.2.10/src/types/pdf_document.rs" />
<option value="$PROJECT_DIR$/Cargo.toml" />
<option value="$PROJECT_DIR$/pkg/ecload/http.go" />
<option value="$PROJECT_DIR$/pkg/ecload/logger.go" />
<option value="$PROJECT_DIR$/cmd/ecload/main.go" />
<option value="$PROJECT_DIR$/cmd/ecload-gui/main.go" />
<option value="$PROJECT_DIR$/LICENSE" />
<option value="$PROJECT_DIR$/.gitignore" />
<option value="$PROJECT_DIR$/pkg/ecload/fetcher.go" />
<option value="$PROJECT_DIR$/Makefile" />
<option value="$PROJECT_DIR$/pkg/ecload/pdf.go" />
<option value="$PROJECT_DIR$/pkg/ecload/ecload.go" />
<option value="$PROJECT_DIR$/README.md" />
<option value="$PROJECT_DIR$/src/main.rs" />
<option value="$PROJECT_DIR$/src/loader.rs" />
<option value="$PROJECT_DIR$/src/error.rs" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="y" value="30" />
<component name="MacroExpansionManager">
<option name="directoryName" value="hGyqkpOy" />
</component>
<component name="ProjectFrameBounds">
<option name="y" value="23" />
<option name="width" value="2488" />
<option name="height" value="1410" />
<option name="height" value="1417" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="PackagesPane" />
<pane id="ProjectPane">
<subPane>
<expand>
@ -106,25 +108,25 @@
<item name="ecload" type="b2602c69:ProjectViewProjectNode" />
<item name="ecload" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="ecload" type="b2602c69:ProjectViewProjectNode" />
<item name="ecload" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="DefaultGoTemplateProperty" value="Go File" />
<property name="android.sdk.path" value="$USER_HOME$/Library/Android/sdk" />
<property name="editor.config.ad.shown" value="true" />
<property name="editorconfig.notification" value="[*.md;*.pdf]&#10;" />
<property name="configurable.Global.GOPATH.is.expanded" value="true" />
<property name="configurable.Module.GOPATH.is.expanded" value="false" />
<property name="configurable.Project.GOPATH.is.expanded" value="true" />
<property name="go.gopath.indexing.explicitly.defined" value="true" />
<property name="go.import.settings.migrated" value="true" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="org.rust.cargo.project.model.PROJECT_DISCOVERY" value="true" />
<property name="settings.editor.selected.configurable" value="preferences.keymap" />
<property name="settings.editor.selected.configurable" value="copyright.profiles" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
@ -138,232 +140,228 @@
</list>
</option>
</component>
<component name="RunManager">
<configuration name="Run ecload" type="CargoCommandRunConfiguration" factoryName="Cargo Command" temporary="true">
<option name="channel" value="DEFAULT" />
<option name="command" value="run --package ecload --bin ecload" />
<option name="allFeatures" value="false" />
<option name="nocapture" value="false" />
<option name="backtrace" value="SHORT" />
<option name="workingDirectory" value="file://$PROJECT_DIR$" />
<envs />
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Cargo Command.Run ecload" />
</list>
</recent_temporary>
<component name="SbtLocalSettings">
<option name="projectSyncType">
<map>
<entry key="$PROJECT_DIR$/../../dev/gitsourcerepositories/dev-ding-10" value="PREVIEW" />
</map>
</option>
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="2503" height="1417" extended-state="6" />
<frame x="0" y="23" width="2488" height="1417" extended-state="0" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.13815522" />
<window_info id="Image Layers" />
<window_info id="Designer" />
<window_info id="Capture Tool" />
<window_info id="Favorites" side_tool="true" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.24979559" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Designer" order="2" />
<window_info id="Favorites" order="3" side_tool="true" />
<window_info anchor="bottom" id="Terminal" visible="true" weight="0.32981133" />
<window_info anchor="bottom" id="Docker" show_stripe_button="false" />
<window_info anchor="bottom" id="Database Changes" />
<window_info anchor="bottom" id="Version Control" />
<window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info active="true" anchor="bottom" id="Run" order="2" visible="true" weight="0.37962264" />
<window_info anchor="bottom" id="Find" order="1" weight="0.32981133" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" order="8" />
<window_info anchor="bottom" id="Database Changes" order="9" />
<window_info anchor="bottom" id="Terminal" order="10" weight="0.32981133" />
<window_info anchor="bottom" id="Event Log" order="11" side_tool="true" />
<window_info anchor="right" id="Theme Preview" />
<window_info anchor="right" id="Maven" />
<window_info anchor="right" id="make" />
<window_info anchor="right" id="Capture Analysis" />
<window_info anchor="right" id="Cargo" />
<window_info anchor="right" id="Palette&#9;" />
<window_info anchor="right" id="SQL Formatter" />
<window_info anchor="right" id="Database" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="right" id="Cargo" order="3" />
<window_info anchor="right" id="Database" order="4" />
<window_info anchor="right" id="Palette&#9;" order="5" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VgoProject">
<integration-enabled>true</integration-enabled>
<proxy>direct</proxy>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/.gitignore">
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/gopkg.in/urfave/cli.v1@v1.20.0/help.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<caret line="5" lean-forward="true" selection-start-line="5" selection-end-line="5" />
<state relative-caret-position="283">
<caret line="172" column="5" selection-start-line="172" selection-start-column="5" selection-end-line="172" selection-end-column="5" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/reqwest-0.9.9/src/error.rs">
<entry file="file://$PROJECT_DIR$/go.mod">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="130">
<caret line="62" column="46" lean-forward="true" selection-start-line="62" selection-start-column="46" selection-end-line="62" selection-end-column="60" />
<state relative-caret-position="45">
<caret line="3" selection-start-line="3" selection-end-line="3" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/reqwest-0.9.9/src/lib.rs">
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/github.com/jawher/mow.cli@v1.1.0/commands.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="145">
<caret line="300" column="39" lean-forward="true" selection-start-line="300" selection-start-column="39" selection-end-line="300" selection-end-column="39" />
<state relative-caret-position="452">
<caret line="648" column="19" selection-start-line="648" selection-start-column="10" selection-end-line="648" selection-end-column="19" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/cssparser-0.24.1/src/lib.rs">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/selectors-0.20.0/lib.rs">
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/github.com/jawher/mow.cli@v1.1.0/cli.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-149" />
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/scraper-0.9.1/src/lib.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="135" column="17" lean-forward="true" selection-start-line="135" selection-start-column="17" selection-end-line="135" selection-end-column="17" />
<state relative-caret-position="400">
<caret line="75" column="23" selection-start-line="75" selection-start-column="23" selection-end-line="75" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/scraper-0.9.1/src/selector.rs">
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/github.com/!puerkito!bio/goquery@v1.5.0/property.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="175">
<caret line="27" column="28" selection-start-line="27" selection-start-column="28" selection-end-line="27" selection-end-column="28" />
<state relative-caret-position="133">
<caret line="15" column="61" selection-start-line="15" selection-start-column="55" selection-end-line="15" selection-end-column="61" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/src/libcore/convert.rs">
<entry file="file://$PROJECT_DIR$/../go/go1.12.5/src/os/types.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="7065">
<caret line="481" column="7" selection-start-line="481" selection-start-column="7" selection-end-line="481" selection-end-column="7" />
<state relative-caret-position="345">
<caret line="26" column="66" selection-start-line="26" selection-start-column="66" selection-end-line="26" selection-end-column="66" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/../go/go1.12.5/src/io/ioutil/ioutil.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="317">
<caret line="97" column="19" lean-forward="true" selection-start-line="97" selection-start-column="19" selection-end-line="97" selection-end-column="19" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/cmd/ecload/main.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" column="4" selection-start-line="14" selection-start-column="4" selection-end-line="14" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pkg/ecload/logger.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="105">
<caret line="7" selection-start-line="7" selection-end-line="7" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/github.com/signintech/gopdf@v0.9.1/config.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270">
<caret line="18" column="5" selection-start-line="18" selection-start-column="5" selection-end-line="18" selection-end-column="5" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/github.com/signintech/gopdf@v0.9.1/page_sizes.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="195">
<caret line="13" column="4" selection-start-line="13" selection-start-column="4" selection-end-line="13" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/github.com/signintech/gopdf@v0.9.1/gopdf.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="287">
<caret line="285" column="17" selection-start-line="285" selection-start-column="17" selection-end-line="285" selection-end-column="17" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/../go/pkg/mod/github.com/jung-kurt/gofpdf@v1.4.2/fpdf.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="280">
<caret line="386" column="15" selection-start-line="386" selection-start-column="15" selection-end-line="386" selection-end-column="15" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/cmd/ecload-gui/main.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" column="32" lean-forward="true" selection-start-line="17" selection-start-column="32" selection-end-line="17" selection-end-column="32" />
<folding>
<element signature="e#15247#15248#0" expanded="true" />
<element signature="e#15280#15281#0" expanded="true" />
<element signature="e#227#280#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/printpdf-0.2.10/src/scale.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="519">
<caret line="55" column="8" selection-start-line="55" selection-start-column="8" selection-end-line="55" selection-end-column="47" />
<folding>
<element signature="e#1296#1297#0" expanded="true" />
<element signature="e#1350#1351#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/image-0.21.0/src/dynimage.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="540">
<caret line="36" column="9" selection-start-line="36" selection-start-column="9" selection-end-line="36" selection-end-column="9" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/image-0.20.1/src/jpeg/decoder.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="258">
<caret line="33" lean-forward="true" selection-start-line="33" selection-end-line="33" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/image-0.20.1/src/dynimage.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="265">
<caret line="716" column="7" selection-start-line="716" selection-start-column="7" selection-end-line="716" selection-end-column="7" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/image-0.20.1/src/image.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="265">
<caret line="220" column="60" lean-forward="true" selection-start-line="220" selection-start-column="60" selection-end-line="220" selection-end-column="60" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/printpdf-0.2.10/src/types/plugins/graphics/xobject.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="220">
<caret line="170" column="5" lean-forward="true" selection-start-line="170" selection-start-column="5" selection-end-line="170" selection-end-column="5" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/printpdf-0.2.10/src/types/plugins/graphics/two_dimensional/image.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="310">
<caret line="32" column="41" selection-start-line="32" selection-start-column="41" selection-end-line="32" selection-end-column="41" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/lopdf-0.17.0/src/processor.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="258">
<caret line="76" column="11" selection-start-line="76" selection-start-column="11" selection-end-line="76" selection-end-column="11" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/printpdf-0.2.10/src/types/pdf_document.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="610">
<caret line="465" column="16" lean-forward="true" selection-start-line="465" selection-start-column="16" selection-end-line="465" selection-end-column="16" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.cargo/registry/src/github.com-1ecc6299db9ec823/clap-2.32.0/src/macros.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="538">
<caret line="60" column="13" selection-start-line="60" selection-start-column="13" selection-end-line="60" selection-end-column="13" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/LICENSE">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="707">
<caret line="373" lean-forward="true" selection-start-line="373" selection-end-line="373" />
<state relative-caret-position="773">
<caret line="373" selection-start-line="373" selection-end-line="373" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/go.sum">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="195">
<caret line="13" column="65" selection-start-line="13" selection-start-column="65" selection-end-line="13" selection-end-column="65" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/.gitignore">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150">
<caret line="10" column="5" selection-start-line="10" selection-start-column="5" selection-end-line="10" selection-end-column="5" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pkg/ecload/fetcher.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-14">
<caret line="17" column="43" selection-start-line="17" selection-start-column="43" selection-end-line="17" selection-end-column="43" />
<folding>
<element signature="e#223#314#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/../go/go1.12.5/src/image/format.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="195">
<caret line="19" column="23" lean-forward="true" selection-start-line="19" selection-start-column="23" selection-end-line="19" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="SPLIT">
<first_editor relative-caret-position="420">
<caret line="28" column="3" lean-forward="true" selection-start-line="28" selection-start-column="3" selection-end-line="28" selection-end-column="3" />
<first_editor relative-caret-position="120">
<caret line="8" column="3" lean-forward="true" selection-start-line="8" selection-start-column="3" selection-end-line="8" selection-end-column="3" />
</first_editor>
<second_editor />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/Cargo.toml">
<entry file="file://$PROJECT_DIR$/Makefile">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="330">
<caret line="22" lean-forward="true" selection-start-line="22" selection-end-line="22" />
<state relative-caret-position="60">
<caret line="4" lean-forward="true" selection-start-line="4" selection-end-line="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main.rs">
<entry file="file://$PROJECT_DIR$/pkg/ecload/pdf.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="355">
<caret line="64" column="23" lean-forward="true" selection-start-line="64" selection-start-column="23" selection-end-line="64" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/loader.rs">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="249">
<caret line="222" column="47" selection-start-line="222" selection-start-column="47" selection-end-line="222" selection-end-column="47" />
<state relative-caret-position="61">
<caret line="5" lean-forward="true" selection-start-line="5" selection-end-line="5" />
<folding>
<element signature="e#4669#4670#0" expanded="true" />
<element signature="e#4723#4724#0" expanded="true" />
<element signature="e#223#291#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/error.rs">
<entry file="file://$PROJECT_DIR$/pkg/ecload/ecload.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="16" column="1" selection-start-line="16" selection-start-column="1" selection-end-line="16" selection-end-column="1" />
<state relative-caret-position="217">
<caret line="31" column="19" lean-forward="true" selection-start-line="31" selection-start-column="19" selection-end-line="31" selection-end-column="19" />
<folding>
<element signature="e#353#354#0" expanded="true" />
<element signature="e#383#384#0" expanded="true" />
<element signature="e#223#321#0" expanded="true" />
</folding>
</state>
</provider>
@ -371,6 +369,18 @@
</component>
<component name="masterDetails">
<states>
<state key="Copyright.UI">
<settings>
<last-edited>MPL-2.0</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8</last-edited>

1994
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

22
Cargo.toml
View File

@ -1,22 +0,0 @@
[package]
name = "ecload"
version = "0.1.0"
authors = ["Sebastian Hugentobler <sebastian@vanwa.ch>"]
edition = "2018"
description = "Download books from https://www.e-codices.unifr.ch"
repository = "https://code.vanwa.ch/sebastian@vanwa.ch/ecload"
readme = "README.md"
keywords = ["e-codice", "cli"]
categories = ["command-line-utilities"]
license = "MPL-2.0"
[dependencies]
clap = "2.32.0"
image = "0.20.0"
log = "0.4.6"
printpdf = "0.2.10"
reqwest = "0.9.9"
scraper = "0.9.1"
simple_logger = "1.0.1"
tempfile = "3.0.5"

7
Makefile Normal file
View File

@ -0,0 +1,7 @@
bin/ecload:
go build -o bin/ecload cmd/ecload/main.go
all: bin/ecload
clean:
rm -r bin/

31
README.md
View File

@ -4,26 +4,31 @@ A small tool to download books from https://www.e-codices.unifr.ch.
# Installation
Install from [crates.io](https://crates.io/crates/ecload).
```
go install code.vanwa.ch/sebastian@vanwa.ch/ecload
```
# Building from source
```
cargo install ecload
make
```
Output is at `bin/ecload`.
# Usage
```
USAGE:
ecload [OPTIONS]
Usage: ecload [-o=<PATH>] [-s=<SIZE>] ID
FLAGS:
-h, --help Prints help information
-V, --version Prints version information
Download books from https://www.e-codices.unifr.ch
Arguments:
ID ID of the book to download. Copy the last two url parts on the overview page to get it (for example bbb/0003).
Options:
-v, --version Show the version and exit
-o, --out-dir save pdf in PATH (default ".")
-s, --size Size of the downloaded images. One of small, medium, large, max. (default "medium")
OPTIONS:
-i, --id <ID> Id of the book to download. Copy the last to url parts on the overview page to get it (for
example bbb/0003).
-o, --out-dir <DIR> The directory where the resulting pdf is saved. [default: .]
-s, --size <SIZE> Sets the size of the downloaded images. [default: medium] [possible values: Small, Medium,
Large, Max]
```

BIN
bin/ecload Executable file
View File

Binary file not shown.

BIN
cmd/ecload/ecload Executable file
View File

Binary file not shown.

60
cmd/ecload/main.go Normal file
View File

@ -0,0 +1,60 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
package main
import (
"io"
"io/ioutil"
"log"
"os"
"github.com/jawher/mow.cli"
"ecload/pkg/ecload"
)
// Initialize logger formats.
func initLogger(
traceHandle io.Writer,
infoHandle io.Writer,
warningHandle io.Writer,
errorHandle io.Writer) ecload.Logger {
return ecload.Logger{
Trace: log.New(traceHandle, "TRACE: ", log.Ldate|log.Ltime),
Info: log.New(infoHandle, "INFO: ", log.Ldate|log.Ltime),
Warning: log.New(warningHandle, "WARNING: ", log.Ldate|log.Ltime),
Error: log.New(errorHandle, "ERROR: ", log.Ldate|log.Ltime),
}
}
func main() {
logger := initLogger(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)
app := cli.App("ecload", "Download books from https://www.e-codices.unifr.ch")
app.Version("v version", "0.1.0")
app.Spec = "[-o=<PATH>] [-s=<SIZE>] ID"
var (
outDir = app.StringOpt("o out-dir", ".", "save pdf in PATH")
size = app.StringOpt("s size", "medium", "Size of the downloaded images. One of small, medium, large, max.")
id = app.StringArg("ID", "", "ID of the book to download. Copy the last two url parts on the overview page to get it (for example bbb/0003).")
)
app.Action = func() {
if *size != "small" && *size != "medium" && *size != "large" && *size != "max" {
logger.Error.Println("SIZE must be one of: small, medium, large, max")
cli.Exit(1)
}
err := ecload.DownloadBook(*outDir, *size, *id, logger)
if err != nil {
logger.Error.Println(err)
}
}
app.Run(os.Args)
}

12
ecload.iml
View File

@ -1,15 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="RUST_MODULE" version="4">
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/examples" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/benches" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<content url="file://$MODULE_DIR$" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

12
go.mod Normal file
View File

@ -0,0 +1,12 @@
module ecload
go 1.12
require (
fyne.io/fyne v1.0.1
github.com/PuerkitoBio/goquery v1.5.0
github.com/jawher/mow.cli v1.1.0
github.com/jung-kurt/gofpdf v1.4.2
github.com/signintech/gopdf v0.9.1
gopkg.in/urfave/cli.v1 v1.20.0
)

53
go.sum Normal file
View File

@ -0,0 +1,53 @@
fyne.io/fyne v1.0.1 h1:f5aD2ZgPdQdDZbQ/UcjNCrv/1/HS4WR3nS0uuYdYmQ4=
fyne.io/fyne v1.0.1/go.mod h1:pA7Zim8v2Ued68/YbB+TAWBpBgfHEgzoMhUoTRGZ/BQ=
github.com/Kodeworks/golang-image-ico v0.0.0-20141118225523-73f0f4cfade9/go.mod h1:7uhhqiBaR4CpN0k9rMjOtjpcfGd6DG2m04zQxKnWQ0I=
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-gl/gl v0.0.0-20181026044259-55b76b7df9d2 h1:78Hza2KHn2PX1jdydQnffaU2A/xM0g3Nx1xmMdep9Gk=
github.com/go-gl/gl v0.0.0-20181026044259-55b76b7df9d2/go.mod h1:482civXOzJJCPzJ4ZOX/pwvXBWSnzD4OKMdH4ClKGbk=
github.com/go-gl/glfw v0.0.0-20181213070059-819e8ce5125f h1:7MsFMbSn8Lcw0blK4+NEOf8DuHoOBDhJsHz04yh13pM=
github.com/go-gl/glfw v0.0.0-20181213070059-819e8ce5125f/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff h1:W71vTCKoxtdXgnm1ECDFkfQnpdqAO00zzGXLA5yaEX8=
github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff/go.mod h1:wfqRWLHRBsRgkp5dmbG56SA0DmVtwrF5N3oPdI8t+Aw=
github.com/jackmordaunt/icns v0.0.0-20181231085925-4f16af745526/go.mod h1:UQkeMHVoNcyXYq9otUupF7/h/2tmHlhrS2zw7ZVvUqc=
github.com/jawher/mow.cli v1.1.0 h1:NdtHXRc0CwZQ507wMvQ/IS+Q3W3x2fycn973/b8Zuk8=
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
github.com/josephspurrier/goversioninfo v0.0.0-20190124120936-8611f5a5ff3f/go.mod h1:eJTEwMjXb7kZ633hO3Ln9mBUCOjX2+FlTljvpl9SYdE=
github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/jung-kurt/gofpdf v1.4.2 h1:3u2ojTwxPPu3ysIOc5iTwcECpvkFCAe2RJ/tQrvfLi0=
github.com/jung-kurt/gofpdf v1.4.2/go.mod h1:rZsO0wEsunjT/L9stF3fJjYbAHgqNYuQB4B8FWvBck0=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
github.com/signintech/gopdf v0.9.1 h1:+lGkICPWQobTA7+I80z52gUHUgtc1/RYKa+nhPcSV1Y=
github.com/signintech/gopdf v0.9.1/go.mod h1:qAvaomt5ROc/Z2zWrEMm64J0238bPhtkpbtlv1WZjtQ=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/srwiley/oksvg v0.0.0-20190105194046-ccbc7673cdf3 h1:8REQ/vZZIZFaZedUSHd7TVkK0CF4heqGtmyz30gCxm8=
github.com/srwiley/oksvg v0.0.0-20190105194046-ccbc7673cdf3/go.mod h1:afMbS0qvv1m5tfENCwnOdZGOF8RGR/FsZ7bvBxQGZG4=
github.com/srwiley/rasterx v0.0.0-20181219215540-696f7edb7a7e h1:FFotfUvew9Eg02LYRl8YybAnm0HCwjjfY5JlOI1oB00=
github.com/srwiley/rasterx v0.0.0-20181219215540-696f7edb7a7e/go.mod h1:mvWM0+15UqyrFKqdRjY6LuAVJR0HOVhJlEgZ5JWtSWU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec h1:arXJwtMuk5vqI1NHX0UTnNw977rYk5Sl4jQqHj+hun4=
golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=

104
pkg/ecload/ecload.go Normal file
View File

@ -0,0 +1,104 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
package ecload
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"io/ioutil"
"os"
"path"
"strings"
"sync"
)
const BASEURL = "https://www.e-codices.unifr.ch"
const THUMBNAILURL = "%s/en/thumbs/%s"
// Save a whole e-codice to the specified directory.
// If the directory does not exist, it will be created.
// size must be one of small, medium, large, max.
func DownloadBook(outDir string, size string, id string, logger Logger) error {
logger.Info.Println("finding pages...")
pageUrls, err := getPageLinks(id)
if err != nil {
return err
}
dir, err := ioutil.TempDir("", "ecload")
if err != nil {
return err
}
defer os.RemoveAll(dir)
var wg sync.WaitGroup
logger.Info.Println("downloading pages...")
for index, pageUrl := range pageUrls {
downloadUrl, err:= getSizeLink(pageUrl, size)
if err != nil {
return err
}
filename := fmt.Sprintf("%06d.jpg", index)
go downloadToFile(filename, dir, downloadUrl, wg)
}
wg.Wait()
err = os.MkdirAll(outDir, os.ModePerm)
if err != nil {
return err
}
pdfPath := path.Join(outDir, fmt.Sprintf("%s.pdf", strings.ReplaceAll(id, "/", "_")))
logger.Info.Printf("Saving pdf to %s...", pdfPath)
return ImgsToPdf(dir, pdfPath)
}
// Find the download link for a page of a specific size.
func getSizeLink(pageUrl string , size string) (string, error) {
doc, err := fetchDocument(pageUrl)
if err != nil {
return "", err
}
downloadSizeUrl := ""
doc.Find("ul.download-page-list > li > a").Each(func(i int, s *goquery.Selection) {
downloadUrl, ok := s.Attr("href")
if ok && strings.HasSuffix(downloadUrl, fmt.Sprintf("/%s", size)) {
downloadSizeUrl = downloadUrl
}
})
return downloadSizeUrl, nil
}
// Find page urls for an e-codice.
// It loads the thumbnail page and parses the html.
func getPageLinks(id string) ([]string, error) {
thumbnailsUrl := fmt.Sprintf(THUMBNAILURL, BASEURL, id)
doc, err := fetchDocument(thumbnailsUrl)
if err != nil {
return nil, err
}
pageUrls := make([]string, 0)
doc.Find("div.thumbnail-image > a").Each(func(i int, s *goquery.Selection) {
pageUrl, ok := s.Attr("href")
if ok {
pageUrls = append(pageUrls, pageUrl)
}
})
return pageUrls, nil
}

68
pkg/ecload/fetcher.go Normal file
View File

@ -0,0 +1,68 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
package ecload
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"io"
"net/http"
"os"
"path"
"sync"
)
// Download a html page from an url (must be UTF-8) and convert it to a goquery document.
func fetchDocument(url string) (*goquery.Document, error) {
res, err := http.Get(url)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
}
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return nil, err
}
return doc, nil
}
// Download a file.
func downloadToFile(filename string, dir string, pageUrl string, wg sync.WaitGroup) error {
wg.Add(1)
defer wg.Done()
fullpath := path.Join(dir, filename)
out, err := os.Create(fullpath)
if err != nil {
return err
}
defer out.Close()
res, err := http.Get(pageUrl)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
}
_, err = io.Copy(out, res.Body)
if err != nil {
return err
}
return nil
}

14
pkg/ecload/logger.go Normal file
View File

@ -0,0 +1,14 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
package ecload
import "log"
type Logger struct {
Trace *log.Logger
Info *log.Logger
Warning *log.Logger
Error *log.Logger
}

51
pkg/ecload/pdf.go Normal file
View File

@ -0,0 +1,51 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
package ecload
import (
"image"
_ "image/jpeg"
"io/ioutil"
"os"
"path"
"github.com/jung-kurt/gofpdf"
)
// Concatenate all jpg files in a directory to a single pdf.
func ImgsToPdf(dir string, output string) error {
pdf := gofpdf.New("P", "mm", "", "")
files, err := ioutil.ReadDir(dir)
if err != nil {
return err
}
for _, f := range files {
filepath := path.Join(dir, f.Name())
reader, err := os.Open(filepath)
if err != nil {
return err
}
img, _, err := image.DecodeConfig(reader)
if err != nil {
//return err
continue
}
pdf.AddPageFormat("P", gofpdf.SizeType{Wd: float64(img.Width), Ht: float64(img.Height)})
opt := gofpdf.ImageOptions{ImageType: "jpg", ReadDpi: true}
pdf.RegisterImageOptionsReader(f.Name(), opt, reader)
pdf.ImageOptions(f.Name(), 0, 0, 0, 0, false, opt, 0, "")
reader.Close()
}
return pdf.OutputFileAndClose(output)
}

41
src/error.rs
View File

@ -1,41 +0,0 @@
use std::fmt;
#[derive(Debug)]
/// Convert different errors into a common one.
pub enum EcloadError {
Error(String),
StdError(std::io::Error),
ReqwestError(reqwest::Error),
PrintPdfError(printpdf::Error),
ImageError(image::ImageError),
}
impl fmt::Display for EcloadError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self)
}
}
impl From<std::io::Error> for EcloadError {
fn from(e: std::io::Error) -> Self {
EcloadError::StdError(e)
}
}
impl From<reqwest::Error> for EcloadError {
fn from(e: reqwest::Error) -> Self {
EcloadError::ReqwestError(e)
}
}
impl From<printpdf::Error> for EcloadError {
fn from(e: printpdf::Error) -> Self {
EcloadError::PrintPdfError(e)
}
}
impl From<image::ImageError> for EcloadError {
fn from(e: image::ImageError) -> Self {
EcloadError::ImageError(e)
}
}

281
src/loader.rs
View File

@ -1,281 +0,0 @@
use std::fs;
use std::fs::DirEntry;
use std::fs::File;
use std::io::BufWriter;
use std::io::copy;
use std::path::Path;
use image::DynamicImage;
use image::GenericImageView;
use printpdf::{Image, Mm, PdfDocument};
use scraper::{Html, Selector};
use tempfile::TempDir;
use crate::error::EcloadError;
const BASEURL: &str = "https://www.e-codices.unifr.ch";
const DPI: f64 = 300.0;
const INCH_AS_MM: f64 = 25.4;
/// Provides functions to download e-codices.
pub struct EcLoader {}
arg_enum! {
#[derive(PartialEq, Debug)]
/// Defines the different sizes that e-codice pages can be.
pub enum PageFormat {
Small,
Medium,
Large,
Max,
}
}
impl PageFormat {
/// Converts the formats in the represantation that the e-codice website uses.
fn as_url_part(&self) -> &str {
match self {
PageFormat::Small => "small",
PageFormat::Medium => "medium",
PageFormat::Large => "large",
PageFormat::Max => "max",
}
}
}
impl EcLoader {
/// Download an e-codice from https://www.e-codices.unifr.ch.
///
/// # Arguments
///
/// * `id` - The combined id ot the e-codice. Consisting of the library id and the e-codice id (for example "bbb/0003").
/// * `title` - Title of the pdf, this will also be the filename.
/// * `out_dir` - Directory where the PDF gets saved (must already exist).
/// * `format` - The size of the page to look for.
pub fn download_codice(
id: &str,
title: &str,
out_dir: &Path,
format: PageFormat,
) -> Result<(), EcloadError> {
let tmp_dir = tempfile::tempdir()?;
debug!("temporary download directory: {}", tmp_dir.path().display());
EcLoader::download_pages(id, format, &tmp_dir)
.and_then(|_| EcLoader::build_pdf(title, out_dir, &tmp_dir))
}
/// Order files in a temporary directory
///
/// # Arguments
///
/// * `tmp_dir`- Reference to the temporary directory.
fn get_ordered_tmp_files(tmp_dir: &TempDir) -> Result<Vec<DirEntry>, EcloadError> {
let paths = fs::read_dir(tmp_dir.path())?;
let mut sorted_paths: Vec<DirEntry> = paths.filter_map(|r| r.ok()).collect();
sorted_paths.sort_by_key(|dir| dir.path());
Ok(sorted_paths)
}
/// Calculate image dimensions in mm.
///
/// # Arguments
///
/// * `image` - Image for the calculation.
fn mm_from_image(image: &DynamicImage) -> (Mm, Mm) {
let (width, height) = image.dimensions();
let mm_width = Mm(width as f64 * (INCH_AS_MM / DPI));
let mm_height = Mm(height as f64 * (INCH_AS_MM / DPI));
(mm_width, mm_height)
}
/// Convert an image to BMP.
///
/// # Arguments
///
/// * `img_path` - Path to the image that should be converted.
///
/// # Remarks
///
/// The BMP file gets saved to the same directory as the original, with the same name apart from
/// the extension.
fn get_img_as_bmp(img_path: &Path) -> Result<DynamicImage, EcloadError> {
let image = image::open(img_path)?;
let img_bmp_path = img_path.with_extension("bmp");
image.save(&img_bmp_path)?;
Ok(image::open(img_bmp_path)?)
}
/// Generate a PDF from images inside a directory.
///
/// # Arguments
///
/// * `title` - Title of the pdf, this will also be the filename.
/// * `out_dir` - Directory where the PDF gets saved (must already exist).
/// * `tmp_dir` - Reference to the temporary directory where the images for the pdf are.
fn build_pdf(title: &str, out_dir: &Path, tmp_dir: &TempDir) -> Result<(), EcloadError> {
let ordered_tmp_files = EcLoader::get_ordered_tmp_files(tmp_dir)?;
let first_file = match ordered_tmp_files.first() {
Some(file) => file,
None => return Err(EcloadError::Error("no tmp files".to_string())),
};
let first_image_data = image::open(first_file.path())?;
let (first_img_width, first_img_height) = EcLoader::mm_from_image(&first_image_data);
let (doc, page1, layer1) =
PdfDocument::new(title, first_img_width, first_img_height, "Layer 1");
let mut current_layer = doc.get_page(page1).get_layer(layer1);
let mut is_first_page = true;
let mut page_count = 1;
for entry in ordered_tmp_files {
let filename_os = entry.file_name();
let filename = match filename_os.to_str() {
Some(filename) => filename,
None => continue,
};
if !filename.ends_with(".jpg") {
continue;
}
info!("saving page {} to pdf...", page_count);
let image_bmp = match EcLoader::get_img_as_bmp(&entry.path()) {
Ok(image_data) => image_data,
Err(err) => {
error!("could not decode {}: {}", entry.path().display(), err);
continue;
}
};
let mut image_file = File::open(entry.path().with_extension("bmp")).unwrap();
let image = Image::try_from(image::bmp::BMPDecoder::new(&mut image_file)).unwrap();
let (img_width, img_height) = EcLoader::mm_from_image(&image_bmp);
debug!("dimensions: {:?} x {:?}", img_width, img_height);
if !is_first_page {
let (new_page, new_layer) = doc.add_page(img_width, img_height, "Layer 1");
current_layer = doc.get_page(new_page).get_layer(new_layer);
}
image.add_to_layer(
current_layer.clone(),
None,
None,
None,
None,
None,
Some(DPI),
);
is_first_page = false;
page_count += 1;
}
let pdf_file_path = out_dir.join(format!("{}.pdf", title));
info!("saving to to {}...", pdf_file_path.display());
let pdf_file = File::create(pdf_file_path)?;
doc.save(&mut BufWriter::new(pdf_file))?;
Ok(())
}
fn get_thumbnail_url(id: &str) -> String {
format!("{}/en/thumbs/{}", BASEURL, id)
}
/// Download all the pages of an e-codice to a temporary directory.
///
/// # Arguments
///
/// * `id` - The combined id ot the e-codice. Consisting of the library id and the e-codice id (for example "bbb/0003").
/// * `format` - The size of the page to look for.
/// * `tmp_dir` - Reference to a temporary directory where the downloaded files get saved.
fn download_pages(id: &str, format: PageFormat, tmp_dir: &TempDir) -> Result<(), EcloadError> {
let thumbnail_url = EcLoader::get_thumbnail_url(id);
let mut response = reqwest::get(thumbnail_url.as_str())?;
let html = response.text()?;
let fragment = Html::parse_fragment(&html);
let li_selector = Selector::parse("div.thumbnail-image > a").unwrap();
let mut page_count = 0;
for element in fragment.select(&li_selector) {
let page_link = match element.value().attr("href") {
Some(href) => href.to_string(),
None => {
continue;
}
};
debug!("searching download links for {}...", page_link);
let download_link = match EcLoader::get_download_link(&page_link, &format) {
Ok(download_link) => download_link,
Err(err) => {
error!(
"could not find download link for {} with format {}: {}",
&page_link, format, err
);
continue;
}
};
debug!("found {}", download_link);
info!("downloading {}...", download_link);
let page_file_path = tmp_dir
.path()
.join(format!("{:0>5}.jpg", page_count.to_string()));
let mut page_download = match reqwest::get(&download_link) {
Ok(response) => response,
Err(err) => {
error!("could not download {}: {}", download_link, err);
continue;
}
};
let mut dest = File::create(page_file_path)?;
copy(&mut page_download, &mut dest)?;
page_count += 1;
}
Ok(())
}
/// Finds the image url for a specified size for an e-codice page.
///
/// # Arguments
///
/// * `page_link` - The specific e-codice page url.
/// * `format` - The size of the page to look for.
fn get_download_link(page_link: &str, format: &PageFormat) -> Result<String, EcloadError> {
let mut response = reqwest::get(page_link)?;
let html = response.text()?;
let fragment = Html::parse_fragment(&html);
let a_selector = Selector::parse("ul.download-page-list > li > a").unwrap();
for element in fragment.select(&a_selector) {
let download_link = match element.value().attr("href") {
Some(href) => href.to_string(),
None => {
continue;
}
};
if download_link.ends_with(format.as_url_part()) {
return Ok(download_link);
}
}
Err(EcloadError::Error(
"could not find download link".to_string(),
))
}
}

86
src/main.rs
View File

@ -1,86 +0,0 @@
#[macro_use]
extern crate clap;
extern crate image;
#[macro_use]
extern crate log;
extern crate printpdf;
extern crate reqwest;
extern crate scraper;
extern crate simple_logger;
extern crate tempfile;
use std::path::Path;
use clap::{App, AppSettings, Arg};
use crate::error::EcloadError;
use crate::loader::EcLoader;
pub mod error;
pub mod loader;
fn main() {
simple_logger::init_with_level(log::Level::Info).unwrap();
let app = App::new("ecload")
.version("0.1.0")
.author("Sebastian Hugentobler <sebastian@vanwa.ch>")
.about("Download books from https://www.e-codices.unifr.ch")
.arg(
Arg::with_name("output directory")
.short("o")
.long("out-dir")
.value_name("DIR")
.help("The directory where the resulting pdf is saved.")
.takes_value(true)
.default_value("."),
)
.arg(
Arg::with_name("size")
.short("s")
.long("size")
.value_name("SIZE")
.help("Sets the size of the downloaded images.")
.takes_value(true)
.possible_values(&loader::PageFormat::variants())
.default_value("medium")
.case_insensitive(true),
)
.arg(
Arg::with_name("id")
.short("i")
.long("id")
.value_name("ID")
.help("Id of the book to download. Copy the last two url parts on the overview page to get it (for example bbb/0003).")
.takes_value(true),
)
.setting(AppSettings::ArgRequiredElseHelp);
let matches = app.clone().get_matches();
let out_path_str = matches.value_of("output directory").unwrap();
let out_path = Path::new(out_path_str);
let size = value_t!(matches.value_of("size"), loader::PageFormat).unwrap();
let id = match matches.value_of("id") {
Some(id) => id,
None => {
println!("ID is a required argument:");
println!();
app.clone().print_help().ok();
println!();
return;
}
};
let title = &id.replace("/", "_");
match run(&out_path, size, id, title) {
Ok(_) => info!("finished!"),
Err(e) => error!("something went wrong: {}", e),
};
}
fn run(out_path: &Path, format: loader::PageFormat, id: &str, title: &str) -> Result<(), EcloadError> {
EcLoader::download_codice(id, title, out_path, format)
}