95 lines
2.5 KiB
JavaScript
95 lines
2.5 KiB
JavaScript
const fs = require("fs");
|
|
const readline = require("readline");
|
|
const { chromium } = require("playwright");
|
|
|
|
const START_URL = "https://watch.lifetimemovieclub.com/movies";
|
|
const OUTPUT_FILE = "lifetimemovieclub_urls.txt";
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function waitForEnter(message) {
|
|
const rl = readline.createInterface({
|
|
input: process.stdin,
|
|
output: process.stdout
|
|
});
|
|
|
|
return new Promise((resolve) => {
|
|
rl.question(message, () => {
|
|
rl.close();
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
async function autoScroll(page, maxRounds = 30, pauseMs = 2500) {
|
|
let lastCount = 0;
|
|
|
|
for (let i = 0; i < maxRounds; i++) {
|
|
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
|
await sleep(pauseMs);
|
|
|
|
const count = await page.locator('a[href*="/movies/"]').count();
|
|
console.log(`Scroll round ${i + 1}: found ${count} matching links`);
|
|
|
|
if (count <= lastCount) {
|
|
console.log("No new links found. Stopping scroll.");
|
|
break;
|
|
}
|
|
|
|
lastCount = count;
|
|
}
|
|
}
|
|
|
|
(async () => {
|
|
const browser = await chromium.launch({
|
|
headless: false,
|
|
slowMo: 50
|
|
});
|
|
|
|
const context = await browser.newContext();
|
|
const page = await context.newPage();
|
|
|
|
try {
|
|
console.log(`Opening ${START_URL}`);
|
|
await page.goto(START_URL, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 60000
|
|
});
|
|
|
|
console.log("Log in in the browser window if needed.");
|
|
await waitForEnter("When the full movies page is visible, press Enter here in PowerShell... ");
|
|
|
|
await page.waitForTimeout(5000);
|
|
|
|
console.log("Starting scroll...");
|
|
await autoScroll(page);
|
|
|
|
const urls = await page.$$eval('a[href*="/movies/"]', (anchors) => {
|
|
return [...new Set(
|
|
anchors
|
|
.map(a => a.href)
|
|
.filter(href => {
|
|
try {
|
|
const u = new URL(href);
|
|
return (
|
|
u.hostname === "watch.lifetimemovieclub.com" &&
|
|
u.pathname.startsWith("/movies/") &&
|
|
u.pathname !== "/movies"
|
|
);
|
|
} catch {
|
|
return false;
|
|
}
|
|
})
|
|
)].sort();
|
|
});
|
|
|
|
fs.writeFileSync(OUTPUT_FILE, urls.join("\n"), "utf8");
|
|
console.log(`Saved ${urls.length} URLs to ${OUTPUT_FILE}`);
|
|
} catch (err) {
|
|
console.error("Error:", err.message);
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
})(); |