140 lines
3.5 KiB
JavaScript
140 lines
3.5 KiB
JavaScript
const fs = require("fs");
|
|
const readline = require("readline");
|
|
const { chromium } = require("playwright");
|
|
|
|
const TARGETS = [
|
|
{
|
|
name: "lifetime",
|
|
url: "https://play.mylifetime.com/movies/lifetime",
|
|
output: "mylifetime_lifetime_urls.txt"
|
|
},
|
|
{
|
|
name: "lmn",
|
|
url: "https://play.mylifetime.com/movies/lmn",
|
|
output: "mylifetime_lmn_urls.txt"
|
|
}
|
|
];
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function waitForEnter(message) {
|
|
const rl = readline.createInterface({
|
|
input: process.stdin,
|
|
output: process.stdout
|
|
});
|
|
|
|
return new Promise((resolve) => {
|
|
rl.question(message, () => {
|
|
rl.close();
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
async function autoScroll(page, maxRounds = 35, pauseMs = 2500) {
|
|
let lastCount = 0;
|
|
|
|
for (let i = 0; i < maxRounds; i++) {
|
|
await page.evaluate(() => {
|
|
window.scrollTo(0, document.body.scrollHeight);
|
|
});
|
|
|
|
await sleep(pauseMs);
|
|
|
|
const count = await page.locator('a[href*="/movies/"]').count();
|
|
console.log(` Scroll round ${i + 1}: found ${count} matching links`);
|
|
|
|
if (count <= lastCount) {
|
|
console.log(" No new links found. Stopping scroll.");
|
|
break;
|
|
}
|
|
|
|
lastCount = count;
|
|
}
|
|
}
|
|
|
|
async function collectMovieUrls(page) {
|
|
const urls = await page.$$eval('a[href*="/movies/"]', (anchors) => {
|
|
return [...new Set(
|
|
anchors
|
|
.map((a) => a.href)
|
|
.filter((href) => {
|
|
try {
|
|
const u = new URL(href);
|
|
return (
|
|
u.hostname === "play.mylifetime.com" &&
|
|
u.pathname.startsWith("/movies/") &&
|
|
u.pathname !== "/movies" &&
|
|
u.pathname !== "/movies/lifetime" &&
|
|
u.pathname !== "/movies/lmn"
|
|
);
|
|
} catch {
|
|
return false;
|
|
}
|
|
})
|
|
)].sort();
|
|
});
|
|
|
|
return urls;
|
|
}
|
|
|
|
(async () => {
|
|
const browser = await chromium.launch({
|
|
headless: false,
|
|
slowMo: 50
|
|
});
|
|
|
|
const context = await browser.newContext();
|
|
const page = await context.newPage();
|
|
|
|
try {
|
|
console.log(`Opening ${TARGETS[0].url}`);
|
|
await page.goto(TARGETS[0].url, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 60000
|
|
});
|
|
|
|
console.log("Log in in the browser window if needed.");
|
|
await waitForEnter("When you're logged in and ready, press Enter here in PowerShell... ");
|
|
|
|
for (const target of TARGETS) {
|
|
console.log(`\nScraping: ${target.name}`);
|
|
console.log(`Opening ${target.url}`);
|
|
|
|
await page.goto(target.url, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 60000
|
|
});
|
|
|
|
await page.waitForTimeout(5000);
|
|
|
|
await autoScroll(page);
|
|
|
|
const urls = await collectMovieUrls(page);
|
|
|
|
fs.writeFileSync(target.output, urls.join("\n"), "utf8");
|
|
console.log(`Saved ${urls.length} URLs to ${target.output}`);
|
|
}
|
|
|
|
const merged = new Set();
|
|
for (const target of TARGETS) {
|
|
const lines = fs.readFileSync(target.output, "utf8")
|
|
.split(/\r?\n/)
|
|
.map((s) => s.trim())
|
|
.filter(Boolean);
|
|
|
|
for (const line of lines) {
|
|
merged.add(line);
|
|
}
|
|
}
|
|
|
|
fs.writeFileSync("mylifetime_all_movie_urls.txt", [...merged].sort().join("\n"), "utf8");
|
|
console.log(`\nSaved ${merged.size} unique combined URLs to mylifetime_all_movie_urls.txt`);
|
|
} catch (err) {
|
|
console.error("Error:", err.message);
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
})(); |