Compare commits

...

2 Commits

Author SHA1 Message Date
Wurzelkoch e55cacd224 Ich glaub ich krieg die Authentifizierung nicht hin 2020-10-10 20:00:22 +02:00
Wurzelkoch 4e0e931839 First guess verification
Warum läuft jetzt der Call Stack über?
2020-10-10 19:27:59 +02:00
1 changed files with 29 additions and 7 deletions

View File

@ -5,6 +5,7 @@ var fs = require('fs');
var plsStop = 0;
var pagesVisited = {};
var printOptions = [];
// set a default starting page (which is the title of Grundgesetz for arbitrary reasons) but take input from CLI
var argv = process.argv.slice(2);
@ -23,11 +24,11 @@ crawl();
function crawl() {
if (plsStop === 1) {
crawl();
return;
}
if (pageToVisit in pagesVisited) {
// We've already visited this page, so repeat the crawl
crawl();
//continue;
} else {
// New page we haven't visited
visitPage(pageToVisit, crawl);
@ -55,14 +56,25 @@ console.log("Visiting page " + url);
// save page
innerhtml = $('div.kapitel');
// console.log("Content: " + innerhtml);
fs.appendFileSync('beckOK.html', innerhtml + '\n');
// download
var options = {
url: baseUrl + "/Print/CurrentDoc" + url + "&printdialogmode=ParentChapter&actionname=&gesamtversionpath=&exportFormat=print",
headers: {
"__RequestVerificationToken": token
}
};
printOptions.push(options);
// prepare next page
var nextPage = $('#next').attr('href');
console.log("Next up: " + nextPage);
if (nextPage){
pageToVisit = baseUrl + nextPage;
pageToVisit = {
url: baseUrl + nextPage,
headers: {
"__RequestVerificationToken": token
}
};
callback();
} else {
plsStop = 1;
@ -71,8 +83,18 @@ console.log("Visiting page " + url);
});
}
function savePage($,callback){
console.log($);
function savePage(callback){
while(printOptions)
var options = printOptions.pop();
request(options, function(error, response, body){
if(response.statusCode === 200) {
// Parse the document body
var $ = cheerio.load(body);
console.log("Page title: " + $('title').text());
innerhtml = $('body');
fs.appendFileSync('beckOK.html', innerhtml + '\n');
};
});
}