I'm trying to use node.js and puppeteer to crawl a website for real-time football information, but after console.log("trying to select team element"), my browser closes immediately.
const puppeteer = require("puppeteer"); async function openPage() { const browser = await puppeteer.launch( {headless: true} ); const page = await browser.newPage(); await page.setViewport({ width: 1000, height: 926 }); await page.goto("https://www.livescore.com/en/"); //Accept cookies const button = await page.waitForSelector('#onetrust-accept-btn-handler'); if (button) { await button.click(); console.log("clicked cookie button"); }; return page; } async function scrapeData(page) { let content = []; // Get the competition elements let elements = await page.waitForSelector(".Ip") console.log("trying to select team element") for (let i=0; i < elements.length; i ) { let homeTeamElement = await elements[i].$(".Ip") if (homeTeamElement) { const homeTeamText = await homeTeamElement.evaluate(node => node.textContent); content.push(homeTeamText); } }; return content; }; (async () => { const page = await openPage(); const dataScraped = await scrapeData(page); console.log(dataScraped) await page.browser().close(); })(); Any ideas as to why this is the case and further criticism of my code are welcome!
![[Web front-end] Node.js quick start](https://img.php.cn/upload/course/000/000/067/662b5d34ba7c0227.png)
await page.waitForSelector(".Ip")Only returns one element, not an array, so it cannot be looped. There should be a clear error message explaining the problem. Instead, usepage.$$eval(or if you want to try the latestlocator API) to extract the data.const puppeteer = require("puppeteer"); // ^21.0.2 const url = "
"; let browser; (async () => { browser = await puppeteer.launch(); const [page] = await browser.pages(); await page.setViewport({width: 1000, height: 926}); await page.goto(url, {waitUntil: "domcontentloaded"}); // not really necessary const button = await page.waitForSelector("#onetrust-accept-btn-handler"); await button.click(); await page.waitForSelector(".Ip"); const content = await page.$$eval(".Ip", els => els.map(e => { const text = id => e.querySelector(`[id*=${id}]`).textContent.trim(); return { time: text("status-or-time"), home: text("home-team-name"), away: text("away-team-name"), homeTeamScore: +text("home-team-score"), awaitTeamScore: +text("away-team-score"), }; }) ); console.log(content); })() .catch(err => console.error(err)) .finally(() => browser?.close());
Note:
waitForSelector. It is guaranteed to be the element, otherwise it will throw an exception if it is not found within the specified time.await elements[i].$(".Ip")won't help you access anything because there is no ## inside the.Ipelement you already hold #.Ip.