shu/nzz-downloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wait until download has startedto go back to issue list

Browse Source
This commit is contained in:
Sebastian Hugentobler 2024-06-25 15:47:06 +02:00
parent 548a4bcf8d
commit aab411fcca
Signed by: shu
GPG Key ID: BB32CF3CA052C2F0
1 changed files with 15 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
nzz.js
View File

@ -14,10 +14,11 @@ const yargs = require("yargs/yargs");
const { hideBin } = require("yargs/helpers"); const { hideBin } = require("yargs/helpers");
const URL = "https://zeitungsarchiv.nzz.ch/"; const URL = "https://zeitungsarchiv.nzz.ch/";
const WAIT_TIMEOUT = 2000; const WAIT_TIMEOUT = 3000;
const TIMEOUT_MSG = `Timeout after ${WAIT_TIMEOUT / 1000} seconds.`; const TIMEOUT_MSG = `Timeout after ${WAIT_TIMEOUT / 1000} seconds.`;
const SEARCH_WAIT_TIMEOUT = 5000; const SEARCH_WAIT_TIMEOUT = 5000;
const SEARCH_TIMEOUT_MSG = `Timeout after ${SEARCH_WAIT_TIMEOUT / 1000} seconds.`; const SEARCH_TIMEOUT_MSG = `Timeout after ${SEARCH_WAIT_TIMEOUT / 1000} seconds.`;
const DOWNLOAD_TIMEOUT = 20000;
const USER_AGENT = const USER_AGENT =
"Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0"; "Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0";
@ -51,23 +52,11 @@ function sleep(ms) {
*/ */
function moveDownload(tmpDir, outDir, date) { function moveDownload(tmpDir, outDir, date) {
return new Promise((resolve) => { return new Promise((resolve) => {
let needsOffset = false;
// Dates in filenames are off by one before 1894
if (date.getFullYear() < 1894) {
date = date.addDays(-1);
needsOffset = true;
}
const srcFile = path.join( const srcFile = path.join(
tmpDir, tmpDir,
`Gesamtausgabe_NZZ_-_Neue_Zürcher_Zeitung_${date.isoDate()}.pdf`, `Gesamtausgabe_NZZ_-_Neue_Zürcher_Zeitung_${date.isoDate()}.pdf`,
); );
if (needsOffset) {
date = date.addDays(1);
}
if (!fs.existsSync(srcFile)) { if (!fs.existsSync(srcFile)) {
setTimeout(() => moveDownload(tmpDir, outDir, date), 2000); setTimeout(() => moveDownload(tmpDir, outDir, date), 2000);
} }
@ -76,7 +65,7 @@ function moveDownload(tmpDir, outDir, date) {
try { try {
fs.copyFileSync(srcFile, destFile); fs.copyFileSync(srcFile, destFile);
} catch { } catch {
// this means it was tried to download a wrong issue // this means we tried to download a wrong issue
} }
resolve(); resolve();
}); });
@ -155,6 +144,7 @@ async function login(driver, user, password) {
await driver.wait(until.elementIsVisible(submitButton), WAIT_TIMEOUT); await driver.wait(until.elementIsVisible(submitButton), WAIT_TIMEOUT);
await submitButton.click(); await submitButton.click();
await driver.switchTo().defaultContent(); await driver.switchTo().defaultContent();
await sleep(500);
const loginMenu = await driver.wait( const loginMenu = await driver.wait(
until.elementLocated( until.elementLocated(
@ -186,12 +176,21 @@ async function download(driver) {
); );
await download.click(); await download.click();
const loadingMask = await driver.wait(
until.elementLocated(By.css(".fup-loading-mask")),
WAIT_TIMEOUT,
TIMEOUT_MSG,
);
await driver.wait(until.elementIsVisible(loadingMask), DOWNLOAD_TIMEOUT);
await driver.wait(until.stalenessOf(loadingMask), DOWNLOAD_TIMEOUT);
const back = await driver.wait( const back = await driver.wait(
until.elementLocated(By.css(".fup-s-menu-back")), until.elementLocated(By.css(".fup-s-menu-back")),
WAIT_TIMEOUT, WAIT_TIMEOUT,
TIMEOUT_MSG, TIMEOUT_MSG,
); );
await driver.wait(until.elementIsVisible(back), WAIT_TIMEOUT); await driver.wait(until.elementIsVisible(back), WAIT_TIMEOUT);
// back.click();
await sleep(500); await sleep(500);
} }
@ -264,8 +263,7 @@ async function run(from, to, user, password, outDir) {
.setPreference("browser.download.folderList", 2) .setPreference("browser.download.folderList", 2)
.setPreference("browser.download.manager.showWhenStartingout", false) .setPreference("browser.download.manager.showWhenStartingout", false)
.setPreference("browser.download.dir", tmpDir.name) .setPreference("browser.download.dir", tmpDir.name)
.setPreference("browser.helperApps.neverAsk.saveToDisk", "application/pdf") .setPreference("browser.helperApps.neverAsk.saveToDisk", "application/pdf");
.setPreference("browser.tabs.warnOnClose", true);
const caps = new Capabilities(); const caps = new Capabilities();
caps.setPageLoadStrategy("normal"); caps.setPageLoadStrategy("normal");
@ -279,7 +277,7 @@ async function run(from, to, user, password, outDir) {
try { try {
await login(driver, user, password); await login(driver, user, password);
await findIssues(driver, from, to, tmpDir.name, outDir); await findIssues(driver, from, to, tmpDir.name, outDir);
// await sleep(1000); await sleep(1000);
} finally { } finally {
await fs.rm(tmpDir.name, { recursive: true }, (e) => { await fs.rm(tmpDir.name, { recursive: true }, (e) => {
if (e) { if (e) {
@ -287,7 +285,6 @@ async function run(from, to, user, password, outDir) {
} }
}); });
await driver.quit(); await driver.quit();
process.exit();
} }
} }