Downloader/lmc-scraper/extract-mylifetime-movie-urls.js

const fs = require("fs");
const readline = require("readline");
const { chromium } = require("playwright");

const TARGETS = [
  {
    name: "lifetime",
    url: "https://play.mylifetime.com/movies/lifetime",
    output: "mylifetime_lifetime_urls.txt"
  },
  {
    name: "lmn",
    url: "https://play.mylifetime.com/movies/lmn",
    output: "mylifetime_lmn_urls.txt"
  }
];

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function waitForEnter(message) {
  const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout
  });

  return new Promise((resolve) => {
    rl.question(message, () => {
      rl.close();
      resolve();
    });
  });
}

async function autoScroll(page, maxRounds = 35, pauseMs = 2500) {
  let lastCount = 0;

  for (let i = 0; i < maxRounds; i++) {
    await page.evaluate(() => {
      window.scrollTo(0, document.body.scrollHeight);
    });

    await sleep(pauseMs);

    const count = await page.locator('a[href*="/movies/"]').count();
    console.log(`  Scroll round ${i + 1}: found ${count} matching links`);

    if (count <= lastCount) {
      console.log("  No new links found. Stopping scroll.");
      break;
    }

    lastCount = count;
  }
}

async function collectMovieUrls(page) {
  const urls = await page.$$eval('a[href*="/movies/"]', (anchors) => {
    return [...new Set(
      anchors
        .map((a) => a.href)
        .filter((href) => {
          try {
            const u = new URL(href);
            return (
              u.hostname === "play.mylifetime.com" &&
              u.pathname.startsWith("/movies/") &&
              u.pathname !== "/movies" &&
              u.pathname !== "/movies/lifetime" &&
              u.pathname !== "/movies/lmn"
            );
          } catch {
            return false;
          }
        })
    )].sort();
  });

  return urls;
}

(async () => {
  const browser = await chromium.launch({
    headless: false,
    slowMo: 50
  });

  const context = await browser.newContext();
  const page = await context.newPage();

  try {
    console.log(`Opening ${TARGETS[0].url}`);
    await page.goto(TARGETS[0].url, {
      waitUntil: "domcontentloaded",
      timeout: 60000
    });

    console.log("Log in in the browser window if needed.");
    await waitForEnter("When you're logged in and ready, press Enter here in PowerShell... ");

    for (const target of TARGETS) {
      console.log(`\nScraping: ${target.name}`);
      console.log(`Opening ${target.url}`);

      await page.goto(target.url, {
        waitUntil: "domcontentloaded",
        timeout: 60000
      });

      await page.waitForTimeout(5000);

      await autoScroll(page);

      const urls = await collectMovieUrls(page);

      fs.writeFileSync(target.output, urls.join("\n"), "utf8");
      console.log(`Saved ${urls.length} URLs to ${target.output}`);
    }

    const merged = new Set();
    for (const target of TARGETS) {
      const lines = fs.readFileSync(target.output, "utf8")
        .split(/\r?\n/)
        .map((s) => s.trim())
        .filter(Boolean);

      for (const line of lines) {
        merged.add(line);
      }
    }

    fs.writeFileSync("mylifetime_all_movie_urls.txt", [...merged].sort().join("\n"), "utf8");
    console.log(`\nSaved ${merged.size} unique combined URLs to mylifetime_all_movie_urls.txt`);
  } catch (err) {
    console.error("Error:", err.message);
  } finally {
    await browser.close();
  }
})();