shu/nzz-downloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wait until download has startedto go back to issue list

Browse Source
This commit is contained in:
Sebastian Hugentobler 2024-06-25 15:47:06 +02:00
parent 548a4bcf8d
commit aab411fcca
Signed by: shu
GPG Key ID: BB32CF3CA052C2F0
1 changed files with 15 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
nzz.js
View File

@ -14,10 +14,11 @@ const yargs = require("yargs/yargs");
const { hideBin } = require("yargs/helpers");
const URL = "https://zeitungsarchiv.nzz.ch/";
const WAIT_TIMEOUT = 2000;
const WAIT_TIMEOUT = 3000;
const TIMEOUT_MSG = `Timeout after ${WAIT_TIMEOUT / 1000} seconds.`;
const SEARCH_WAIT_TIMEOUT = 5000;
const SEARCH_TIMEOUT_MSG = `Timeout after ${SEARCH_WAIT_TIMEOUT / 1000} seconds.`;
const DOWNLOAD_TIMEOUT = 20000;
const USER_AGENT =
"Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0";
@ -51,23 +52,11 @@ function sleep(ms) {
*/
function moveDownload(tmpDir, outDir, date) {
return new Promise((resolve) => {
let needsOffset = false;
// Dates in filenames are off by one before 1894
if (date.getFullYear() < 1894) {
date = date.addDays(-1);
needsOffset = true;
}
const srcFile = path.join(
tmpDir,
`Gesamtausgabe_NZZ_-_Neue_Zürcher_Zeitung_${date.isoDate()}.pdf`,
);
if (needsOffset) {
date = date.addDays(1);
}
if (!fs.existsSync(srcFile)) {
setTimeout(() => moveDownload(tmpDir, outDir, date), 2000);
}
@ -76,7 +65,7 @@ function moveDownload(tmpDir, outDir, date) {
try {
fs.copyFileSync(srcFile, destFile);
} catch {
// this means it was tried to download a wrong issue
// this means we tried to download a wrong issue
}
resolve();
});
@ -155,6 +144,7 @@ async function login(driver, user, password) {
await driver.wait(until.elementIsVisible(submitButton), WAIT_TIMEOUT);
await submitButton.click();
await driver.switchTo().defaultContent();
await sleep(500);
const loginMenu = await driver.wait(
until.elementLocated(
@ -186,12 +176,21 @@ async function download(driver) {
);
await download.click();
const loadingMask = await driver.wait(
until.elementLocated(By.css(".fup-loading-mask")),
WAIT_TIMEOUT,
TIMEOUT_MSG,
);
await driver.wait(until.elementIsVisible(loadingMask), DOWNLOAD_TIMEOUT);
await driver.wait(until.stalenessOf(loadingMask), DOWNLOAD_TIMEOUT);
const back = await driver.wait(
until.elementLocated(By.css(".fup-s-menu-back")),
WAIT_TIMEOUT,
TIMEOUT_MSG,
);
await driver.wait(until.elementIsVisible(back), WAIT_TIMEOUT);
// back.click();
await sleep(500);
}
@ -264,8 +263,7 @@ async function run(from, to, user, password, outDir) {
.setPreference("browser.download.folderList", 2)
.setPreference("browser.download.manager.showWhenStartingout", false)
.setPreference("browser.download.dir", tmpDir.name)
.setPreference("browser.helperApps.neverAsk.saveToDisk", "application/pdf")
.setPreference("browser.tabs.warnOnClose", true);
.setPreference("browser.helperApps.neverAsk.saveToDisk", "application/pdf");
const caps = new Capabilities();
caps.setPageLoadStrategy("normal");
@ -279,7 +277,7 @@ async function run(from, to, user, password, outDir) {
try {
await login(driver, user, password);
await findIssues(driver, from, to, tmpDir.name, outDir);
// await sleep(1000);
await sleep(1000);
} finally {
await fs.rm(tmpDir.name, { recursive: true }, (e) => {
if (e) {
@ -287,7 +285,6 @@ async function run(from, to, user, password, outDir) {
}
});
await driver.quit();
process.exit();
}
}