From fd124c828dc692c81354a196865c67b368193923 Mon Sep 17 00:00:00 2001
From: Bommels05 <69975756+Bommels05@users.noreply.github.com>
Date: Fri, 17 Apr 2026 18:03:04 +0200
Subject: [PATCH] HTTP Proxy & More

---
 build.gradle                                  |   2 +
 .../bommels05/befatorweb/BefatorRewriter.java |  54 +++++---
 .../de/bommels05/befatorweb/BefatorWeb.java   | 119 ++++++++++++------
 .../de/bommels05/befatorweb/SavedPaged.java   |   2 +-
 .../de/bommels05/befatorweb/SiteCache.java    |  17 ++-
 .../befatorweb/links/LinkDatabase.java        |  31 +++--
 6 files changed, 148 insertions(+), 77 deletions(-)

diff --git a/build.gradle b/build.gradle
index b99c068..563cf06 100644
--- a/build.gradle
+++ b/build.gradle
@@ -21,6 +21,8 @@ dependencies {
     implementation("com.google.code.gson:gson:2.10.1")
     implementation("com.github.ben-manes.caffeine:caffeine:3.2.2")
     implementation("org.jsoup:jsoup:1.21.1")
+    implementation("org.graphstream:gs-core:2.0")
+    implementation("org.graphstream:gs-ui-swing:2.0")
 
     implementation("it.uniroma1.dis.wsngroup.gexf4j:gexf4j:1.0.0")
 
diff --git a/src/main/java/de/bommels05/befatorweb/BefatorRewriter.java b/src/main/java/de/bommels05/befatorweb/BefatorRewriter.java
index 3d0de45..8ebaf53 100644
--- a/src/main/java/de/bommels05/befatorweb/BefatorRewriter.java
+++ b/src/main/java/de/bommels05/befatorweb/BefatorRewriter.java
@@ -9,6 +9,8 @@ import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.function.Consumer;
 import java.util.regex.Matcher;
@@ -35,7 +37,7 @@ public class BefatorRewriter {
                 if (href != null) {
                     Attribute rel = link.attribute("rel");
                     if (rel != null) {
-                        String newLink = rewriteLink(href.getValue());
+                        String newLink = rewriteLink(href.getValue(), url);
                         if (rel.getValue().equalsIgnoreCase("stylesheet")) {
                             LOGGER.info("Rewriting Stylesheet Link: {} -> {}", href.getValue(), newLink);
                             href.setValue(newLink);
@@ -56,7 +58,7 @@ public class BefatorRewriter {
                     if (type != null) {
                         if (type.getValue().equalsIgnoreCase("refresh")) {
                             String link = content.getValue().substring(content.getValue().indexOf("=") + 1);
-                            String newLink = rewriteLink(link);
+                            String newLink = rewriteLink(link, url);
                             LOGGER.info("Rewriting <meta> refresh Link: {} -> {}", link, newLink);
                             content.setValue(content.getValue().replaceFirst(link, newLink));
                         }
@@ -68,7 +70,7 @@ public class BefatorRewriter {
             for (Element base : bases) {
                 Attribute href = base.attribute("href");
                 if (href != null) {
-                    String newLink = rewriteLink(href.getValue());
+                    String newLink = rewriteLink(href.getValue(), url);
                     LOGGER.info("Rewriting <base> Link: {} -> {}", href.getValue(), newLink);
                     href.setValue(newLink);
                 }
@@ -78,7 +80,7 @@ public class BefatorRewriter {
             for (Element script : scripts) {
                 Attribute src = script.attribute("src");
                 if (src != null) {
-                    String newLink = rewriteLink(src.getValue());
+                    String newLink = rewriteLink(src.getValue(), url);
                     LOGGER.info("Rewriting <script> Link: {} -> {}", src.getValue(), newLink);
                     src.setValue(newLink);
                 }
@@ -88,7 +90,7 @@ public class BefatorRewriter {
             for (Element img : images) {
                 Attribute src = img.attribute("src");
                 if (src != null) {
-                    String newLink = rewriteLink(src.getValue());
+                    String newLink = rewriteLink(src.getValue(), url);
                     LOGGER.info("Rewriting <img> Link: {} -> {}", src.getValue(), newLink);
                     src.setValue(newLink);
                 }
@@ -98,7 +100,7 @@ public class BefatorRewriter {
             for (Element link : links) {
                 Attribute href = link.attribute("href");
                 if (href != null) {
-                    String newLink = rewriteLink(href.getValue());
+                    String newLink = rewriteLink(href.getValue(), url);
                     LOGGER.info("Rewriting <a> Link: {} -> {}", href.getValue(), newLink);
                     href.setValue(newLink);
                     LinkDatabase.addLink(url, newLink);
@@ -109,7 +111,7 @@ public class BefatorRewriter {
             for (Element iframe : iframes) {
                 Attribute src = iframe.attribute("src");
                 if (src != null) {
-                    String newLink = rewriteLink(src.getValue());
+                    String newLink = rewriteLink(src.getValue(), url);
                     LOGGER.info("Rewriting <iframe> Link: {} -> {}", src.getValue(), newLink);
                     src.setValue(newLink);
                 }
@@ -121,20 +123,20 @@ public class BefatorRewriter {
             }
 
             if (!foundIcon) {
-                html.head().append("<link rel=\"icon\" href=\"/proxy/" + (url.contains("/") ? url.substring(0, url.indexOf("/")) : url) + "/favicon.ico\">");
+                //html.head().append("<link rel=\"icon\" href=\"/proxy/" + (url.contains("/") ? url.substring(0, url.indexOf("/")) : url) + "/favicon.ico\">");
             }
             input = html.toString().getBytes(StandardCharsets.UTF_8);
         } else if (contentType == SavedPaged.ContentType.CSS) {
             Matcher matcher =  Pattern.compile("url\\([\"']?(.*?)[\"']?\\)").matcher(inputString);
             while (matcher.find()) {
-                String newLink = rewriteLink(matcher.group(1));
+                String newLink = rewriteLink(matcher.group(1), url);
                 LOGGER.info("Rewriting CSS url() Link: {} -> {}", matcher.group(1), newLink);
                 inputString = inputString.replace(matcher.group(), "url(" + newLink + ")");
             }
 
             matcher = Pattern.compile("@import \"(.*?)\"").matcher(inputString);
             while (matcher.find()) {
-                String newLink = rewriteLink(matcher.group(1));
+                String newLink = rewriteLink(matcher.group(1), url);
                 LOGGER.info("Rewriting CSS @import Link: {} -> {}", matcher.group(1), newLink);
                 inputString = inputString.replace(matcher.group(), "@import \"" + newLink + "\"");
             }
@@ -146,7 +148,7 @@ public class BefatorRewriter {
             while (inputString.contains("\"https://web.archive.org/web/")) {
                 int i = inputString.indexOf("\"https://web.archive.org/web/");
                 String link = inputString.substring(i + 1, inputString.indexOf("\"", i + 1));
-                String newLink = rewriteLink(link);
+                String newLink = rewriteLink(link, url);
                 LOGGER.info("Rewriting undetected Link: {} -> {}", link, newLink);
                 inputString = inputString.replace("\"" + link + "\"", newLink);
             }
@@ -155,32 +157,44 @@ public class BefatorRewriter {
         return input;
     }
 
-    public static String rewriteLink(String input) {
+    public static String rewriteLink(String input, String host) {
         if (input.startsWith("https://web-static.archive.org")) {
             return input/*.replace("https://web-static.archive.org", "/invalid")*/;
         }
         if (input.startsWith("https://web.archive.org")) {
-            return rewriteLink(input.replace("https://web.archive.org", ""));
+            return rewriteLink(input.replace("https://web.archive.org", ""), host);
         }
 
         if (input.startsWith("/web/")) {
             String s = input.replaceFirst("/web/", "");
             String original = s.substring(s.indexOf("/") + 1);
 
-            String newLink = rewriteLink(original);
-            if (!newLink.startsWith("/proxy/")) {
-                newLink = "/proxy/" + newLink;
-            }
-            return newLink;
+            return rewriteLink(original, host);
         }
 
-        if (input.startsWith("http://")) {
+        /*if (input.startsWith("http://")) {
             return input.replaceFirst("http://", "/proxy/");
         }
         if (input.startsWith("https://")) {
             return input.replaceFirst("https://", "/proxy/");
+        }*/
+
+        if (host != null) {
+            try {
+                URI base = new URI("http://" + host);
+                URI relative = new URI(input);
+                URI result = base.resolve(relative);
+                return result.toString();
+            } catch (URISyntaxException e) {
+                if (input.startsWith("mailto:")) {
+                    return input;
+                }
+                e.printStackTrace();
+                return input;
+            }
+        } else {
+            return input;
         }
-        return input;
     }
 
     public static boolean sameTargets(String link1, String link2) {
diff --git a/src/main/java/de/bommels05/befatorweb/BefatorWeb.java b/src/main/java/de/bommels05/befatorweb/BefatorWeb.java
index aff8e68..aa89afd 100644
--- a/src/main/java/de/bommels05/befatorweb/BefatorWeb.java
+++ b/src/main/java/de/bommels05/befatorweb/BefatorWeb.java
@@ -6,11 +6,23 @@ import com.google.gson.Gson;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
+import de.bommels05.befatorweb.gui.FileBrowserWindow;
+import de.bommels05.befatorweb.links.LinkDBEntry;
 import de.bommels05.befatorweb.links.LinkDatabase;
 import io.javalin.Javalin;
+import org.graphstream.graph.Graph;
+import org.graphstream.graph.Node;
+import org.graphstream.graph.implementations.SingleGraph;
+import org.graphstream.ui.geom.Point2;
+import org.graphstream.ui.geom.Point3;
+import org.graphstream.ui.view.Viewer;
+import org.graphstream.ui.view.camera.Camera;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.awt.*;
+import java.awt.event.MouseWheelEvent;
+import java.awt.event.MouseWheelListener;
 import java.io.IOException;
 import java.net.URL;
 import java.net.URLConnection;
@@ -26,46 +38,24 @@ public class BefatorWeb {
     public static final int TARGET_YEAR = 2004;
     public static final int TARGET_MONTH = Calendar.OCTOBER;
     public static final int REQUEST_DELAY = 5000;
-    public static final String TARGET_URL = "www.microsoft.com/"/*"www.mozilla.org/releases/mozilla1.0/"*/;
+    public static final String TARGET_URL = "www.jetbrains.com/index.html"/*"www.microsoft.com/"/*"www.mozilla.org/releases/mozilla1.0/"*/;
+    public static final boolean SHOW_MISSING = true;
 
     private static final Logger LOGGER = LoggerFactory.getLogger(BefatorWeb.class);
     private static final LoadingCache<String, String> urlCache = Caffeine.newBuilder().build(BefatorWeb::getClosestArchiveUrl);
     private static final ReentrantLock requestLock = new ReentrantLock();
+    private static final Graph graph = new SingleGraph("Befator Web");
 
     public static void main(String[] args) {
-        Javalin app = Javalin.create().start(3636);
+        System.setProperty("org.graphstream.ui", "swing");
+        Javalin app = Javalin.create().start("0.0.0.0", 3636);
 
-        app.get("/", ctx -> {
-            ctx.result("Befator Inc grüßt Sie!");
-        });
-
-        app.get("/api/progressMessage", ctx -> {
-            String target = ctx.queryParam("targetPage") == null ? TARGET_URL : ctx.queryParam("targetPage");
-            String url = ctx.queryParam("currentPage");
-            if (url == null) {
-                ctx.result("Dein aktuelles Ziel ist: " + target);
-            } else {
-                int distance = LinkDatabase.getDistance(url, target);
-                if (distance == -1) {
-                    ctx.result("Es ist (noch) kein sicherer Weg zu deinem Ziel (" + target + ") bekannt");
-                } else {
-                    ctx.result("Du bist " + distance + " Links von deinem Ziel (" + target + ") entfernt");
-                }
+        app.before(ctx -> {
+            String url = ctx.host() + ctx.path();
+            /*if (ctx.path().startsWith("/proxy/")) {
+                url = ctx.path().replaceFirst("/proxy/", "");
             }
-        });
-
-        /*app.before(ctx -> {
-            String path = ctx.path();
-
-            if (!path.replaceFirst("/proxy/", "").contains("/")/* && !path.matches(".*\\.[a-zA-Z0-9]+$")*//*) {
-                String query = ctx.queryString() != null ? "?" + ctx.queryString() : "";
-                ctx.redirect(path + "/" + query);
-            }
-        });*/
-
-        app.get("/proxy/*", ctx -> {
-            String url = BefatorRewriter.stripProtocol(ctx.path().replaceFirst("/proxy/", ""));
-            /*if (url.endsWith("/")) {
+            if (url.endsWith("/")) {
                 url = url.substring(0, url.length() - 1);
             }*/
             url = url + (ctx.queryString() != null ? "?" + ctx.queryString() : "");
@@ -92,8 +82,14 @@ public class BefatorWeb {
                         SavedPaged.ContentType contentType = content.contentType();
                         LOGGER.info("Downloaded page {} from {} with type {}", url, archiveUrl, contentType);
                         SiteCache.addPage(url, content.content(), contentType, content.status());
+
+                        Node node = graph.getNode(url);
+                        if (node != null) {
+                            node.setAttribute("ui.style", "fill-color: gray;");
+                        }
                     }
                 } finally {
+                    Thread.sleep(REQUEST_DELAY);
                     requestLock.unlock();
                 }
             }
@@ -101,14 +97,67 @@ public class BefatorWeb {
             SavedPaged page = SiteCache.getPage(url);
 
             if (page.status() == SavedPaged.StatusCode.REDIRECT) {
-                ctx.redirect(BefatorRewriter.rewriteLink(new String(page.getContent(), StandardCharsets.UTF_8)));
+                ctx.redirect(BefatorRewriter.rewriteLink(new String(page.getContent(), StandardCharsets.UTF_8), url));
                 return;
             }
 
             ctx.result(page.getContent());
             ctx.res().setCharacterEncoding(StandardCharsets.UTF_8.name());
             ctx.contentType(page.type().toString());
+            ctx.skipRemainingHandlers();
         });
+
+        app.get("/api/progressMessage", ctx -> {
+            String target = ctx.queryParam("targetPage") == null ? TARGET_URL : ctx.queryParam("targetPage");
+            String url = ctx.queryParam("currentPage");
+            if (url == null) {
+                ctx.result("Dein aktuelles Ziel ist: " + target);
+            } else {
+                int distance = LinkDatabase.getDistance(url, target);
+                if (distance == -1) {
+                    ctx.result("Es ist (noch) kein sicherer Weg zu deinem Ziel (" + target + ") bekannt");
+                } else {
+                    ctx.result("Du bist " + distance + " Links von deinem Ziel (" + target + ") entfernt");
+                }
+            }
+        });
+
+        Viewer viewer = graph.display();
+        viewer.getDefaultView().enableMouseOptions();
+        ((Component) viewer.getDefaultView()).addMouseWheelListener(new MouseWheelListener() {
+            @Override
+            public void mouseWheelMoved(MouseWheelEvent e) {
+                e.consume();
+                int i = e.getWheelRotation();
+                double factor = Math.pow(1.25, i);
+                Camera cam = viewer.getDefaultView().getCamera();
+                double zoom = cam.getViewPercent() * factor;
+                Point2 pxCenter  = cam.transformGuToPx(cam.getViewCenter().x, cam.getViewCenter().y, 0);
+                Point3 guClicked = cam.transformPxToGu(e.getX(), e.getY());
+                double newRatioPx2Gu = cam.getMetrics().ratioPx2Gu/factor;
+                double x = guClicked.x + (pxCenter.x - e.getX())/newRatioPx2Gu;
+                double y = guClicked.y - (pxCenter.y - e.getY())/newRatioPx2Gu;
+                cam.setViewCenter(x, y, 0);
+                cam.setViewPercent(zoom);
+            }
+        });
+        LinkDatabase.LINKS_TABLE.getAll().forEach(BefatorWeb::addLinkToGraph);
+    }
+
+    public static void addLinkToGraph(LinkDBEntry link) {
+        if (graph.getNode(link.getSource()) == null) {
+            Node node = graph.addNode(link.getSource());
+            node.setAttribute("ui.label", link.getSource());
+            node.setAttribute("ui.style", SiteCache.containsPage(link.getSource()) ? "fill-color: gray;" : "fill-color: red;");
+        }
+        if (SHOW_MISSING || SiteCache.containsPage(link.getDestination())) {
+            if (graph.getNode(link.getDestination()) == null) {
+                Node node = graph.addNode(link.getDestination());
+                node.setAttribute("ui.label", link.getDestination());
+                node.setAttribute("ui.style", SiteCache.containsPage(link.getDestination()) ? "fill-color: gray;" : "fill-color: red;");
+            }
+            graph.addEdge(link.getSource() + "-" + link.getDestination(), link.getSource(), link.getDestination(), true);
+        }
     }
 
     private static PageDownloadData downloadPage(String url) {
@@ -120,7 +169,7 @@ public class BefatorWeb {
 
             String finalLocation = connection.getURL().toString();
             if (!BefatorRewriter.sameTargets(finalLocation, url)) {
-                LinkDatabase.addLink(BefatorRewriter.rewriteLink(url), BefatorRewriter.rewriteLink(finalLocation));
+                LinkDatabase.addLink(BefatorRewriter.rewriteLink(url, null), BefatorRewriter.rewriteLink(finalLocation, null));
                 return new PageDownloadData(finalLocation.getBytes(StandardCharsets.UTF_8), SavedPaged.ContentType.REDIRECT, SavedPaged.StatusCode.REDIRECT);
             } else {
                 return new PageDownloadData(connection.getInputStream().readAllBytes(), SavedPaged.ContentType.fromString(connection.getContentType()), SavedPaged.StatusCode.OK);
@@ -176,7 +225,7 @@ public class BefatorWeb {
 
                 JsonObject dayInfo = getJsonContent("https://web.archive.org/__wb/calendarcaptures/2?url=" + encoded + "&date=" + targetYear + toTwoDigitString(calendar.get(Calendar.MONTH) + 1) + toTwoDigitString(calendar.get(Calendar.DAY_OF_MONTH)));
                 String timeOfDay = dayInfo.get("items").getAsJsonArray().get(0).getAsJsonArray().get(0).getAsString();
-                timeOfDay = timeOfDay.length() == 5 ? "0" + timeOfDay : timeOfDay;
+                timeOfDay = timeOfDay.length() == 4 ? "00" + timeOfDay : timeOfDay.length() == 5 ? "0" + timeOfDay : timeOfDay;
                 return "https://web.archive.org/web/" + targetYear + toTwoDigitString(calendar.get(Calendar.MONTH) + 1) + toTwoDigitString(calendar.get(Calendar.DAY_OF_MONTH)) + timeOfDay + "/" + url;
             }
         }
diff --git a/src/main/java/de/bommels05/befatorweb/SavedPaged.java b/src/main/java/de/bommels05/befatorweb/SavedPaged.java
index 225f1df..b17e697 100644
--- a/src/main/java/de/bommels05/befatorweb/SavedPaged.java
+++ b/src/main/java/de/bommels05/befatorweb/SavedPaged.java
@@ -3,7 +3,7 @@ package de.bommels05.befatorweb;
 public record SavedPaged(String url, ContentType type, StatusCode status) {
 
     public byte[] getContent() {
-        return SiteCache.getPageContent(url);
+        return SiteCache.getPageContent(url, type);
     }
 
     public enum ContentType {
diff --git a/src/main/java/de/bommels05/befatorweb/SiteCache.java b/src/main/java/de/bommels05/befatorweb/SiteCache.java
index a93d11f..7923964 100644
--- a/src/main/java/de/bommels05/befatorweb/SiteCache.java
+++ b/src/main/java/de/bommels05/befatorweb/SiteCache.java
@@ -14,12 +14,14 @@ import java.util.ArrayList;
 import java.util.List;
 
 public class SiteCache {
+    public static final String CACHE_FOLDER = "storage";
+
     private static final Logger LOGGER = LoggerFactory.getLogger(SiteCache.class);
     private static final List<SavedPaged> pages = new ArrayList<>();
     private static final BefatorRewriter rewriter = new BefatorRewriter(url -> {});
     static {
         try {
-            File cache = new File("cache");
+            File cache = new File(CACHE_FOLDER);
             if (!cache.exists()) {
                 Files.createDirectory(cache.toPath());
             }
@@ -45,20 +47,24 @@ public class SiteCache {
         return getPage(url) != null;
     }
 
-    public static byte[] getPageContent(String url) {
-        return readFile("cache/" + pages.indexOf(getPage(url)));
+    public static byte[] getPageContent(String url, SavedPaged.ContentType type) {
+        return readFile(CACHE_FOLDER + "/" + pages.indexOf(getPage(url)));
     }
 
     public static void addPage(String url, byte[] content, SavedPaged.ContentType contentType, SavedPaged.StatusCode status) {
         if (!containsPage(url)) {
             pages.add(new SavedPaged(url, contentType, status));
-            overwriteFile("cache/" + pages.indexOf(getPage(url)), rewriter.rewrite(content, url, contentType));
+            overwriteFile(CACHE_FOLDER + "/" + pages.indexOf(getPage(url)), rewriter.rewrite(content, url, contentType));
             saveIndex();
         } else {
             throw new IllegalArgumentException("Page " + url + " is already cached");
         }
     }
 
+    public static List<SavedPaged> getPages() {
+        return pages;
+    }
+
     private static byte[] readFile(String fileName) {
         try {
             FileInputStream stream = new FileInputStream(fileName);
@@ -78,6 +84,9 @@ public class SiteCache {
         try {
             File file = new File(fileName);
             Files.deleteIfExists(file.toPath());
+            if (file.getParentFile() != null) {
+                Files.createDirectories(file.getParentFile().toPath());
+            }
 
             FileOutputStream stream = new FileOutputStream(file);
             stream.write(content);
diff --git a/src/main/java/de/bommels05/befatorweb/links/LinkDatabase.java b/src/main/java/de/bommels05/befatorweb/links/LinkDatabase.java
index 4efd245..8085c67 100644
--- a/src/main/java/de/bommels05/befatorweb/links/LinkDatabase.java
+++ b/src/main/java/de/bommels05/befatorweb/links/LinkDatabase.java
@@ -1,6 +1,7 @@
 package de.bommels05.befatorweb.links;
 
 import de.bommels05.befatorweb.BefatorRewriter;
+import de.bommels05.befatorweb.BefatorWeb;
 import de.bommels05.befatorweb.links.calc.BefatorDistanceCalculator;
 import de.bommels05.befatorweb.links.calc.DistanceCalculator;
 import de.bommels05.befatorweb.links.calc.V2DistanceCalculator;
@@ -33,33 +34,29 @@ public class LinkDatabase {
     public static void addLink(String source, String destination) {
         if (destination.startsWith("#")) {
             return;
+        } else if (destination.substring(destination.indexOf("/") + 1).contains("#")) {
+            destination = destination.substring(0, destination.lastIndexOf("#"));
         }
 
-        if (source.startsWith("/proxy/")) {
-            source = source.replaceFirst("/proxy/", "");
-        }
-        if (destination.startsWith("/proxy/")) {
-            destination = destination.replaceFirst("/proxy/", "https://");
+        if (destination.startsWith("http://")) {
+            destination = destination.replaceFirst("http://", "");
         }
 
-        try {
-            URI base = new URI(source);
-            URI relative = new URI(destination);
-            URI result = base.resolve(relative);
-
-            String finalDestination = result.toString();
-            if (finalDestination.startsWith("https://")) {
+        //try {
+            /*if (finalDestination.startsWith("https://")) {
                 finalDestination = finalDestination.replaceFirst("https://", "");
-            }
+            }*/
 
-            if (source.equals(finalDestination) || hasDirectLink(source, finalDestination)) {
+            if (source.equals(destination) || hasDirectLink(source, destination)) {
                 return;
             }
 
-            new LinkDBEntry(LINKS_TABLE, source, finalDestination).save();
-        } catch (URISyntaxException e) {
+            LinkDBEntry link = new LinkDBEntry(LINKS_TABLE, source, destination);
+            link.save();
+            BefatorWeb.addLinkToGraph(link);
+        /*} catch (URISyntaxException e) {
             LOGGER.error("Invalid Link URL", e);
-        }
+        }*/
     }
 
     private static boolean hasDirectLink(String source, String destination) {