shu/nzz-downloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

set loading strategy explicitly

Browse Source
This commit is contained in:
Sebastian Hugentobler 2020-12-23 22:58:47 +01:00
parent 17c4589b51
commit 1993e0b049
Signed by: shu
GPG Key ID: BB32CF3CA052C2F0
2 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -53,5 +53,8 @@ directory "./nzz"
You need a good internet connection, as the program only waits 5 seconds until a
download of an issue can start. This is something that is hard to solve unfortunately.
If you get strange errors about elements not being visible, wait a bit and try again,
it's usually a network problem.
## Licence
Licenced as [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/).

20
nzz.js
View File

@ -8,7 +8,7 @@
const tmp = require('tmp');
var path = require('path');
const fs = require('fs');
const { Builder, By, Key, until } = require('selenium-webdriver');
const { Builder, By, Capabilities, Key, until } = require('selenium-webdriver');
const fx = require('selenium-webdriver/firefox');
const yargs = require('yargs/yargs')
const { hideBin } = require('yargs/helpers');
@ -59,7 +59,7 @@ async function moveDownload(tmpDir, outDir, date) {
}
let srcFile = path.join(tmpDir, `Gesamtausgabe_NZZ_-_Neue_Zürcher_Zeitung_${date.isoDate()}.pdf`);
fs.watchFile(srcFile, () => {
if (fs.existsSync(srcFile)) {
@ -67,10 +67,10 @@ async function moveDownload(tmpDir, outDir, date) {
if (needsOffset) {
date = date.addDays(1);
}
let destFile = path.join(outDir, `${date.isoDate()}.pdf`);
fs.copyFileSync(srcFile, destFile);
resolve();
}
});
@ -86,11 +86,11 @@ async function moveDownload(tmpDir, outDir, date) {
async function enterDate(driver, date) {
let dateString = date.nzzDate();
let startDate = await driver.wait(until.elementLocated(By.css('.fup-s-date-start')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY);
let startDate = await driver.wait(until.elementLocated(By.css('input.fup-s-date-start')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY);
await startDate.clear();
await startDate.sendKeys(dateString);
let endDate = await driver.wait(until.elementLocated(By.css('.fup-s-date-end')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY);
let endDate = await driver.wait(until.elementLocated(By.css('input.fup-s-date-end')), WAIT_TIMEOUT, TIMEOUT_MSG, RETRY_DELAY);
await endDate.clear();
await endDate.sendKeys(dateString + Key.ENTER);
}
@ -155,8 +155,8 @@ async function findIssues(driver, from, to, tmpDir, outDir) {
try {
let articles = await driver.wait(until.elementsLocated(By.css('.fup-archive-result-item-article-title')), SEARCH_WAIT_TIMEOUT, SEARCH_TIMEOUT_MSG, RETRY_DELAY);
await articles[0].click();s
await articles[0].click();
await download(driver);
console.log(`downloading ${from.isoDate()}...`);
@ -198,7 +198,11 @@ async function run(from, to, user, password, outDir) {
.setPreference('browser.download.dir', tmpDir.name)
.setPreference('browser.helperApps.neverAsk.saveToDisk', 'application/pdf');
const caps = new Capabilities();
caps.setPageLoadStrategy("normal");
let driver = await new Builder()
.withCapabilities(caps)
.forBrowser('firefox')
.setFirefoxOptions(fxOptions)
.build();