I am creating a Chrome extension to help me collect search results, but I am facing a problem in this part
function extract_urls(domainInput) {
var links = document.querySelectorAll('a[ping]');
var counter = 0;
var link, domain;
for (var i = 12; i < links.length; i++) {
link = links[i].getAttribute("ping").split("url=")[1];
domain = new URL(link).hostname;
if (domain.endsWith(domainInput)) {
counter++;
endpoints.push(link);
}
}
return counter;
}
function scraper(domainInput){
var counter = extract_urls(domainInput);
var start = 1;
while (true) {
window.location.href="https://www.google.com/search?q=site:" + domainInput + '&num=100&start=" + start;
start++;
counter = extract_urls(domainInput);
if (counter == 0){
break;
}
}
console.log(endpoints);
}
var endpoints = [];
It is assumed that you are redirected to Google search, then extract_urls is executed to extract the results and save them in the endpoints until it reaches a page that does not contain results and the loop is exited, but the problem here is that it repeats your redirection to Google without waiting for the page to complete loading and then extract_urls is executed
I tried many methods, but I can”t stop the code
Have you tried wrapping your code in a DOMContentLoaded event listener callback ?