diff --git a/crawler.js b/crawler.js index 7a69ae1..4d59616 100644 --- a/crawler.js +++ b/crawler.js @@ -35,9 +35,10 @@ if (plsStop === 1) { } -function visitPage(pageToVisit, callback){ -console.log("Visiting page " + pageToVisit); - request(pageToVisit, function(error, response, body) { +function visitPage(url, callback){ +console.log("Visiting page " + url); + pagesVisited[url] = true; + request(url, function(error, response, body) { if(error) { console.log("Error: " + error); } @@ -49,8 +50,7 @@ console.log("Visiting page " + pageToVisit); console.log("Page title: " + $('title').text()); // savePage($,crawl); // prepare next page - console.log("Next up: " + $("#dk2next").text()); - var nextPage = $('#dk2next').attr('href'); + var nextPage = $('#next').attr('href'); console.log("Next up: " + nextPage); if (nextPage){ pageToVisit = baseUrl + nextPage;