README updated

README.md

@@ -0,0 +1,15 @@
+# BeckOK
+
+Der Beck'sche OnlineKommentar ist nur mit Account nutzbar und zeigt den Kommentar nur Paragraphen- oder gar randnummerweise an. Das wollen wir ändern: BeckOK soll einmal aus dem Link zur Titelseite eines Gesetzbuchs ein PDF mit dem gesamten Buch machen.
+
+# Systemvorraussetzungen
+`node.js` und `npm`.
+
+Run with `node crawler.js <START_URL>`. Wird START\_URL nicht angegeben, wird die Titelseite des Grundgesetzes verwendet.
+
+# Stand
+Das Skript blättert alle aufeinanderfolgend frei zugänglichen Seiten ab der angegebenen durch, ohne sie abzuspeichern.
+
+# Todos 
+* Abspeichern, obviously
+* evtl. Authentifizierung?

crawler.js

@@ -2,38 +2,68 @@ var request = require('request');
 var cheerio = require('cheerio');
 var URL = require('url-parse');
 
-var pageToVisit = "https://beck-online.beck.de/?vpath=bibdata/komm/BeckOKGG_44/cont/BECKOKGG.glUD1.htm";
-console.log("Visiting page " + pageToVisit);
-request(pageToVisit, function(error, response, body) {
-   if(error) {
-     console.log("Error: " + error);
-   }
-   // Check status code (200 is HTTP OK)
-   console.log("Status code: " + response.statusCode);
-   if(response.statusCode === 200) {
-     // Parse the document body
-     var $ = cheerio.load(body);
-     console.log("Page title:  " + $('#dk2next').attr('href'));
-     collectInternalLinks($);
-   }
-});
+var plsStop = 0;
+var pagesVisited = {};
 
-function collectInternalLinks($) {
-  var allRelativeLinks = [];
-  var allAbsoluteLinks = [];
+// set a default starting page (which is the title of Grundgesetz for arbitrary reasons) but take input from CLI
+var argv = process.argv.slice(2);
+var START_URL = "https://beck-online.beck.de/?vpath=bibdata/komm/BeckOKGG_44/cont/BECKOKGG.glUD1.htm";
+if (argv[0]){
+    START_URL = argv[0];
+}
+console.log(START_URL);
+var url = new URL(START_URL);
+var baseUrl = url.protocol + "//" + url.hostname;
 
-  var relativeLinks = $("a[href^='/']");
-  relativeLinks.each(function() {
-      allRelativeLinks.push($(this).attr('href'));
+pageToVisit = START_URL;
+crawl();
 
-  });
+// function declarations:
 
-  var absoluteLinks = $("a[href^='http']");
-  absoluteLinks.each(function() {
-      allAbsoluteLinks.push($(this).attr('href'));
-  });
-
-  console.log("Found " + allRelativeLinks.length + " relative links");
-  console.log("Found " + allAbsoluteLinks.length + " absolute links");
+function crawl() {
+console.log(plsStop);
+if (plsStop === 1) {
+	crawl();
+}
+  if (pageToVisit in pagesVisited) {
+    // We've already visited this page, so repeat the crawl
+    crawl();
+  } else {
+    // New page we haven't visited
+    visitPage(pageToVisit, crawl);
+  }
 }
 
+
+function visitPage(url, callback){
+console.log("Visiting page " + url);
+    pagesVisited[url] = true;
+    request(url, function(error, response, body) {
+       if(error) {
+         console.log("Error: " + error);
+       }
+       // Check status code (200 is HTTP OK)
+       console.log("Status code: " + response.statusCode);
+       if(response.statusCode === 200) {
+         // Parse the document body
+         var $ = cheerio.load(body);
+         console.log("Page title:  " + $('title').text());
+//	       savePage($,crawl);
+         // prepare next page
+         var nextPage = $('#next').attr('href');
+	       console.log("Next up: " + nextPage);
+         if (nextPage){
+             pageToVisit = baseUrl + nextPage;
+ 	     callback();
+         } else {
+             plsStop = 1;
+         }
+       }
+    });
+}
+
+function savePage($,callback){
+	console.log($);
+}
+
+