shu/nzz-downloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

set loading strategy explicitly

Browse Source
This commit is contained in:
Sebastian Hugentobler 2020-12-23 22:58:47 +01:00
parent 17c4589b51
commit 1993e0b049
Signed by: shu
GPG Key ID: BB32CF3CA052C2F0
2 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -53,5 +53,8 @@ directory "./nzz"
You need a good internet connection, as the program only waits 5 seconds until a You need a good internet connection, as the program only waits 5 seconds until a
download of an issue can start. This is something that is hard to solve unfortunately. download of an issue can start. This is something that is hard to solve unfortunately.
If you get strange errors about elements not being visible, wait a bit and try again,
it's usually a network problem.
## Licence ## Licence
Licenced as [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/). Licenced as [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/).

20
nzz.js
View File

@ -8,7 +8,7 @@
const tmp = require('tmp'); const tmp = require('tmp');
var path = require('path'); var path = require('path');
const fs = require('fs'); const fs = require('fs');
const { Builder, By, Key, until } = require('selenium-webdriver'); const { Builder, By, Capabilities, Key, until } = require('selenium-webdriver');
const fx = require('selenium-webdriver/firefox'); const fx = require('selenium-webdriver/firefox');
const yargs = require('yargs/yargs') const yargs = require('yargs/yargs')
const { hideBin } = require('yargs/helpers'); const { hideBin } = require('yargs/helpers');
@ -59,7 +59,7 @@ async function moveDownload(tmpDir, outDir, date) {
} }
let srcFile = path.join(tmpDir, `Gesamtausgabe_NZZ_-_Neue_Zürcher_Zeitung_${date.isoDate()}.pdf`); let srcFile = path.join(tmpDir, `Gesamtausgabe_NZZ_-_Neue_Zürcher_Zeitung_${date.isoDate()}.pdf`);
fs.watchFile(srcFile, () => { fs.watchFile(srcFile, () => {
if (fs.existsSync(srcFile)) { if (fs.existsSync(srcFile)) {
@ -67,10 +67,10 @@ async function moveDownload(tmpDir, outDir, date) {
if (needsOffset) { if (needsOffset) {
date = date.addDays(1); date = date.addDays(1);
} }
let destFile = path.join(outDir, `${date.isoDate()}.pdf`); let destFile = path.join(outDir, `${date.isoDate()}.pdf`);
fs.copyFileSync(srcFile, destFile); fs.copyFileSync(srcFile, destFile);
resolve(); resolve();
} }
}); });
@ -86,11 +86,11 @@ async function moveDownload(tmpDir, outDir, date) {
async function enterDate(driver, date) { async function enterDate(driver, date) {
let dateString = date.nzzDate(); let dateString = date.nzzDate();
let startDate = await driver.wait(until.elementLocated(By.css('.fup-s-date-start')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY); let startDate = await driver.wait(until.elementLocated(By.css('input.fup-s-date-start')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY);
await startDate.clear(); await startDate.clear();
await startDate.sendKeys(dateString); await startDate.sendKeys(dateString);
let endDate = await driver.wait(until.elementLocated(By.css('.fup-s-date-end')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY); let endDate = await driver.wait(until.elementLocated(By.css('input.fup-s-date-end')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY);
await endDate.clear(); await endDate.clear();
await endDate.sendKeys(dateString + Key.ENTER); await endDate.sendKeys(dateString + Key.ENTER);
} }
@ -155,8 +155,8 @@ async function findIssues(driver, from, to, tmpDir, outDir) {
try { try {
let articles = await driver.wait(until.elementsLocated(By.css('.fup-archive-result-item-article-title')), SEARCH_WAIT_TIMEOUT, SEARCH_TIMEOUT_MSG, RETRY_DELAY); let articles = await driver.wait(until.elementsLocated(By.css('.fup-archive-result-item-article-title')), SEARCH_WAIT_TIMEOUT, SEARCH_TIMEOUT_MSG, RETRY_DELAY);
await articles[0].click();s await articles[0].click();
await download(driver); await download(driver);
console.log(`downloading ${from.isoDate()}...`); console.log(`downloading ${from.isoDate()}...`);
@ -198,7 +198,11 @@ async function run(from, to, user, password, outDir) {
.setPreference('browser.download.dir', tmpDir.name) .setPreference('browser.download.dir', tmpDir.name)
.setPreference('browser.helperApps.neverAsk.saveToDisk', 'application/pdf'); .setPreference('browser.helperApps.neverAsk.saveToDisk', 'application/pdf');
const caps = new Capabilities();
caps.setPageLoadStrategy("normal");
let driver = await new Builder() let driver = await new Builder()
.withCapabilities(caps)
.forBrowser('firefox') .forBrowser('firefox')
.setFirefoxOptions(fxOptions) .setFirefoxOptions(fxOptions)
.build(); .build();