Bommels05/BefatorWeb
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HTTP Proxy & More

Browse Source
This commit is contained in:
Bommels05 committed 2026-04-17 18:03:04 +02:00
1 parent 28ffb254a3
commit fd124c828d
6 files changed
+148 -77
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

build.gradle
+2
View File
@@ -21,6 +21,8 @@ dependencies {
implementation("com.google.code.gson:gson:2.10.1")
implementation("com.github.ben-manes.caffeine:caffeine:3.2.2")
implementation("org.jsoup:jsoup:1.21.1")
implementation("org.graphstream:gs-core:2.0")
implementation("org.graphstream:gs-ui-swing:2.0")
implementation("it.uniroma1.dis.wsngroup.gexf4j:gexf4j:1.0.0")
src/main/java/de/bommels05/befatorweb/BefatorRewriter.java
+34 -20
View File
@@ -9,6 +9,8 @@ import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.function.Consumer;
import java.util.regex.Matcher;
@@ -35,7 +37,7 @@ public class BefatorRewriter {
if (href != null) {
Attribute rel = link.attribute("rel");
if (rel != null) {
String newLink = rewriteLink(href.getValue());
String newLink = rewriteLink(href.getValue(), url);
if (rel.getValue().equalsIgnoreCase("stylesheet")) {
LOGGER.info("Rewriting Stylesheet Link: {} -> {}", href.getValue(), newLink);
href.setValue(newLink);
@@ -56,7 +58,7 @@ public class BefatorRewriter {
if (type != null) {
if (type.getValue().equalsIgnoreCase("refresh")) {
String link = content.getValue().substring(content.getValue().indexOf("=") + 1);
String newLink = rewriteLink(link);
String newLink = rewriteLink(link, url);
LOGGER.info("Rewriting <meta> refresh Link: {} -> {}", link, newLink);
content.setValue(content.getValue().replaceFirst(link, newLink));
}
@@ -68,7 +70,7 @@ public class BefatorRewriter {
for (Element base : bases) {
Attribute href = base.attribute("href");
if (href != null) {
String newLink = rewriteLink(href.getValue());
String newLink = rewriteLink(href.getValue(), url);
LOGGER.info("Rewriting <base> Link: {} -> {}", href.getValue(), newLink);
href.setValue(newLink);
}
@@ -78,7 +80,7 @@ public class BefatorRewriter {
for (Element script : scripts) {
Attribute src = script.attribute("src");
if (src != null) {
String newLink = rewriteLink(src.getValue());
String newLink = rewriteLink(src.getValue(), url);
LOGGER.info("Rewriting <script> Link: {} -> {}", src.getValue(), newLink);
src.setValue(newLink);
}
@@ -88,7 +90,7 @@ public class BefatorRewriter {
for (Element img : images) {
Attribute src = img.attribute("src");
if (src != null) {
String newLink = rewriteLink(src.getValue());
String newLink = rewriteLink(src.getValue(), url);
LOGGER.info("Rewriting <img> Link: {} -> {}", src.getValue(), newLink);
src.setValue(newLink);
}
@@ -98,7 +100,7 @@ public class BefatorRewriter {
for (Element link : links) {
Attribute href = link.attribute("href");
if (href != null) {
String newLink = rewriteLink(href.getValue());
String newLink = rewriteLink(href.getValue(), url);
LOGGER.info("Rewriting <a> Link: {} -> {}", href.getValue(), newLink);
href.setValue(newLink);
LinkDatabase.addLink(url, newLink);
@@ -109,7 +111,7 @@ public class BefatorRewriter {
for (Element iframe : iframes) {
Attribute src = iframe.attribute("src");
if (src != null) {
String newLink = rewriteLink(src.getValue());
String newLink = rewriteLink(src.getValue(), url);
LOGGER.info("Rewriting <iframe> Link: {} -> {}", src.getValue(), newLink);
src.setValue(newLink);
}
@@ -121,20 +123,20 @@ public class BefatorRewriter {
}
if (!foundIcon) {
html.head().append("<link rel=\"icon\" href=\"/proxy/" + (url.contains("/") ? url.substring(0, url.indexOf("/")) : url) + "/favicon.ico\">");
//html.head().append("<link rel=\"icon\" href=\"/proxy/" + (url.contains("/") ? url.substring(0, url.indexOf("/")) : url) + "/favicon.ico\">");
}
input = html.toString().getBytes(StandardCharsets.UTF_8);
} else if (contentType == SavedPaged.ContentType.CSS) {
Matcher matcher = Pattern.compile("url\\([\"']?(.*?)[\"']?\\)").matcher(inputString);
while (matcher.find()) {
String newLink = rewriteLink(matcher.group(1));
String newLink = rewriteLink(matcher.group(1), url);
LOGGER.info("Rewriting CSS url() Link: {} -> {}", matcher.group(1), newLink);
inputString = inputString.replace(matcher.group(), "url(" + newLink + ")");
}
matcher = Pattern.compile("@import \"(.*?)\"").matcher(inputString);
while (matcher.find()) {
String newLink = rewriteLink(matcher.group(1));
String newLink = rewriteLink(matcher.group(1), url);
LOGGER.info("Rewriting CSS @import Link: {} -> {}", matcher.group(1), newLink);
inputString = inputString.replace(matcher.group(), "@import \"" + newLink + "\"");
}
@@ -146,7 +148,7 @@ public class BefatorRewriter {
while (inputString.contains("\"https://web.archive.org/web/")) {
int i = inputString.indexOf("\"https://web.archive.org/web/");
String link = inputString.substring(i + 1, inputString.indexOf("\"", i + 1));
String newLink = rewriteLink(link);
String newLink = rewriteLink(link, url);
LOGGER.info("Rewriting undetected Link: {} -> {}", link, newLink);
inputString = inputString.replace("\"" + link + "\"", newLink);
}
@@ -155,32 +157,44 @@ public class BefatorRewriter {
return input;
}
public static String rewriteLink(String input) {
public static String rewriteLink(String input, String host) {
if (input.startsWith("https://web-static.archive.org")) {
return input/*.replace("https://web-static.archive.org", "/invalid")*/;
}
if (input.startsWith("https://web.archive.org")) {
return rewriteLink(input.replace("https://web.archive.org", ""));
return rewriteLink(input.replace("https://web.archive.org", ""), host);
}
if (input.startsWith("/web/")) {
String s = input.replaceFirst("/web/", "");
String original = s.substring(s.indexOf("/") + 1);
String newLink = rewriteLink(original);
if (!newLink.startsWith("/proxy/")) {
newLink = "/proxy/" + newLink;
}
return newLink;
return rewriteLink(original, host);
}
if (input.startsWith("http://")) {
/*if (input.startsWith("http://")) {
return input.replaceFirst("http://", "/proxy/");
}
if (input.startsWith("https://")) {
return input.replaceFirst("https://", "/proxy/");
}*/
if (host != null) {
try {
URI base = new URI("http://" + host);
URI relative = new URI(input);
URI result = base.resolve(relative);
return result.toString();
} catch (URISyntaxException e) {
if (input.startsWith("mailto:")) {
return input;
}
e.printStackTrace();
return input;
}
} else {
return input;
}
return input;
}
public static boolean sameTargets(String link1, String link2) {
src/main/java/de/bommels05/befatorweb/BefatorWeb.java
+84 -35
View File
@@ -6,11 +6,23 @@ import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import de.bommels05.befatorweb.gui.FileBrowserWindow;
import de.bommels05.befatorweb.links.LinkDBEntry;
import de.bommels05.befatorweb.links.LinkDatabase;
import io.javalin.Javalin;
import org.graphstream.graph.Graph;
import org.graphstream.graph.Node;
import org.graphstream.graph.implementations.SingleGraph;
import org.graphstream.ui.geom.Point2;
import org.graphstream.ui.geom.Point3;
import org.graphstream.ui.view.Viewer;
import org.graphstream.ui.view.camera.Camera;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.awt.*;
import java.awt.event.MouseWheelEvent;
import java.awt.event.MouseWheelListener;
import java.io.IOException;
import java.net.URL;
import java.net.URLConnection;
@@ -26,46 +38,24 @@ public class BefatorWeb {
public static final int TARGET_YEAR = 2004;
public static final int TARGET_MONTH = Calendar.OCTOBER;
public static final int REQUEST_DELAY = 5000;
public static final String TARGET_URL = "www.microsoft.com/"/*"www.mozilla.org/releases/mozilla1.0/"*/;
public static final String TARGET_URL = "www.jetbrains.com/index.html"/*"www.microsoft.com/"/*"www.mozilla.org/releases/mozilla1.0/"*/;
public static final boolean SHOW_MISSING = true;
private static final Logger LOGGER = LoggerFactory.getLogger(BefatorWeb.class);
private static final LoadingCache<String, String> urlCache = Caffeine.newBuilder().build(BefatorWeb::getClosestArchiveUrl);
private static final ReentrantLock requestLock = new ReentrantLock();
private static final Graph graph = new SingleGraph("Befator Web");
public static void main(String[] args) {
Javalin app = Javalin.create().start(3636);
System.setProperty("org.graphstream.ui", "swing");
Javalin app = Javalin.create().start("0.0.0.0", 3636);
app.get("/", ctx -> {
ctx.result("Befator Inc grüßt Sie!");
});
app.get("/api/progressMessage", ctx -> {
String target = ctx.queryParam("targetPage") == null ? TARGET_URL : ctx.queryParam("targetPage");
String url = ctx.queryParam("currentPage");
if (url == null) {
ctx.result("Dein aktuelles Ziel ist: " + target);
} else {
int distance = LinkDatabase.getDistance(url, target);
if (distance == -1) {
ctx.result("Es ist (noch) kein sicherer Weg zu deinem Ziel (" + target + ") bekannt");
} else {
ctx.result("Du bist " + distance + " Links von deinem Ziel (" + target + ") entfernt");
}
app.before(ctx -> {
String url = ctx.host() + ctx.path();
/*if (ctx.path().startsWith("/proxy/")) {
url = ctx.path().replaceFirst("/proxy/", "");
}
});
/*app.before(ctx -> {
String path = ctx.path();
if (!path.replaceFirst("/proxy/", "").contains("/")/* && !path.matches(".*\\.[a-zA-Z0-9]+$")*//*) {
String query = ctx.queryString() != null ? "?" + ctx.queryString() : "";
ctx.redirect(path + "/" + query);
}
});*/
app.get("/proxy/*", ctx -> {
String url = BefatorRewriter.stripProtocol(ctx.path().replaceFirst("/proxy/", ""));
/*if (url.endsWith("/")) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}*/
url = url + (ctx.queryString() != null ? "?" + ctx.queryString() : "");
@@ -92,8 +82,14 @@ public class BefatorWeb {
SavedPaged.ContentType contentType = content.contentType();
LOGGER.info("Downloaded page {} from {} with type {}", url, archiveUrl, contentType);
SiteCache.addPage(url, content.content(), contentType, content.status());
Node node = graph.getNode(url);
if (node != null) {
node.setAttribute("ui.style", "fill-color: gray;");
}
}
} finally {
Thread.sleep(REQUEST_DELAY);
requestLock.unlock();
}
}
@@ -101,14 +97,67 @@ public class BefatorWeb {
SavedPaged page = SiteCache.getPage(url);
if (page.status() == SavedPaged.StatusCode.REDIRECT) {
ctx.redirect(BefatorRewriter.rewriteLink(new String(page.getContent(), StandardCharsets.UTF_8)));
ctx.redirect(BefatorRewriter.rewriteLink(new String(page.getContent(), StandardCharsets.UTF_8), url));
return;
}
ctx.result(page.getContent());
ctx.res().setCharacterEncoding(StandardCharsets.UTF_8.name());
ctx.contentType(page.type().toString());
ctx.skipRemainingHandlers();
});
app.get("/api/progressMessage", ctx -> {
String target = ctx.queryParam("targetPage") == null ? TARGET_URL : ctx.queryParam("targetPage");
String url = ctx.queryParam("currentPage");
if (url == null) {
ctx.result("Dein aktuelles Ziel ist: " + target);
} else {
int distance = LinkDatabase.getDistance(url, target);
if (distance == -1) {
ctx.result("Es ist (noch) kein sicherer Weg zu deinem Ziel (" + target + ") bekannt");
} else {
ctx.result("Du bist " + distance + " Links von deinem Ziel (" + target + ") entfernt");
}
}
});
Viewer viewer = graph.display();
viewer.getDefaultView().enableMouseOptions();
((Component) viewer.getDefaultView()).addMouseWheelListener(new MouseWheelListener() {
@Override
public void mouseWheelMoved(MouseWheelEvent e) {
e.consume();
int i = e.getWheelRotation();
double factor = Math.pow(1.25, i);
Camera cam = viewer.getDefaultView().getCamera();
double zoom = cam.getViewPercent() * factor;
Point2 pxCenter = cam.transformGuToPx(cam.getViewCenter().x, cam.getViewCenter().y, 0);
Point3 guClicked = cam.transformPxToGu(e.getX(), e.getY());
double newRatioPx2Gu = cam.getMetrics().ratioPx2Gu/factor;
double x = guClicked.x + (pxCenter.x - e.getX())/newRatioPx2Gu;
double y = guClicked.y - (pxCenter.y - e.getY())/newRatioPx2Gu;
cam.setViewCenter(x, y, 0);
cam.setViewPercent(zoom);
}
});
LinkDatabase.LINKS_TABLE.getAll().forEach(BefatorWeb::addLinkToGraph);
}
public static void addLinkToGraph(LinkDBEntry link) {
if (graph.getNode(link.getSource()) == null) {
Node node = graph.addNode(link.getSource());
node.setAttribute("ui.label", link.getSource());
node.setAttribute("ui.style", SiteCache.containsPage(link.getSource()) ? "fill-color: gray;" : "fill-color: red;");
}
if (SHOW_MISSING || SiteCache.containsPage(link.getDestination())) {
if (graph.getNode(link.getDestination()) == null) {
Node node = graph.addNode(link.getDestination());
node.setAttribute("ui.label", link.getDestination());
node.setAttribute("ui.style", SiteCache.containsPage(link.getDestination()) ? "fill-color: gray;" : "fill-color: red;");
}
graph.addEdge(link.getSource() + "-" + link.getDestination(), link.getSource(), link.getDestination(), true);
}
}
private static PageDownloadData downloadPage(String url) {
@@ -120,7 +169,7 @@ public class BefatorWeb {
String finalLocation = connection.getURL().toString();
if (!BefatorRewriter.sameTargets(finalLocation, url)) {
LinkDatabase.addLink(BefatorRewriter.rewriteLink(url), BefatorRewriter.rewriteLink(finalLocation));
LinkDatabase.addLink(BefatorRewriter.rewriteLink(url, null), BefatorRewriter.rewriteLink(finalLocation, null));
return new PageDownloadData(finalLocation.getBytes(StandardCharsets.UTF_8), SavedPaged.ContentType.REDIRECT, SavedPaged.StatusCode.REDIRECT);
} else {
return new PageDownloadData(connection.getInputStream().readAllBytes(), SavedPaged.ContentType.fromString(connection.getContentType()), SavedPaged.StatusCode.OK);
@@ -176,7 +225,7 @@ public class BefatorWeb {
JsonObject dayInfo = getJsonContent("https://web.archive.org/__wb/calendarcaptures/2?url=" + encoded + "&date=" + targetYear + toTwoDigitString(calendar.get(Calendar.MONTH) + 1) + toTwoDigitString(calendar.get(Calendar.DAY_OF_MONTH)));
String timeOfDay = dayInfo.get("items").getAsJsonArray().get(0).getAsJsonArray().get(0).getAsString();
timeOfDay = timeOfDay.length() == 5 ? "0" + timeOfDay : timeOfDay;
timeOfDay = timeOfDay.length() == 4 ? "00" + timeOfDay : timeOfDay.length() == 5 ? "0" + timeOfDay : timeOfDay;
return "https://web.archive.org/web/" + targetYear + toTwoDigitString(calendar.get(Calendar.MONTH) + 1) + toTwoDigitString(calendar.get(Calendar.DAY_OF_MONTH)) + timeOfDay + "/" + url;
}
}
src/main/java/de/bommels05/befatorweb/SavedPaged.java
+1 -1
View File
@@ -3,7 +3,7 @@ package de.bommels05.befatorweb;
public record SavedPaged(String url, ContentType type, StatusCode status) {
public byte[] getContent() {
return SiteCache.getPageContent(url);
return SiteCache.getPageContent(url, type);
}
public enum ContentType {
src/main/java/de/bommels05/befatorweb/SiteCache.java
+13 -4
View File
@@ -14,12 +14,14 @@ import java.util.ArrayList;
import java.util.List;
public class SiteCache {
public static final String CACHE_FOLDER = "storage";
private static final Logger LOGGER = LoggerFactory.getLogger(SiteCache.class);
private static final List<SavedPaged> pages = new ArrayList<>();
private static final BefatorRewriter rewriter = new BefatorRewriter(url -> {});
static {
try {
File cache = new File("cache");
File cache = new File(CACHE_FOLDER);
if (!cache.exists()) {
Files.createDirectory(cache.toPath());
}
@@ -45,20 +47,24 @@ public class SiteCache {
return getPage(url) != null;
}
public static byte[] getPageContent(String url) {
return readFile("cache/" + pages.indexOf(getPage(url)));
public static byte[] getPageContent(String url, SavedPaged.ContentType type) {
return readFile(CACHE_FOLDER + "/" + pages.indexOf(getPage(url)));
}
public static void addPage(String url, byte[] content, SavedPaged.ContentType contentType, SavedPaged.StatusCode status) {
if (!containsPage(url)) {
pages.add(new SavedPaged(url, contentType, status));
overwriteFile("cache/" + pages.indexOf(getPage(url)), rewriter.rewrite(content, url, contentType));
overwriteFile(CACHE_FOLDER + "/" + pages.indexOf(getPage(url)), rewriter.rewrite(content, url, contentType));
saveIndex();
} else {
throw new IllegalArgumentException("Page " + url + " is already cached");
}
}
public static List<SavedPaged> getPages() {
return pages;
}
private static byte[] readFile(String fileName) {
try {
FileInputStream stream = new FileInputStream(fileName);
@@ -78,6 +84,9 @@ public class SiteCache {
try {
File file = new File(fileName);
Files.deleteIfExists(file.toPath());
if (file.getParentFile() != null) {
Files.createDirectories(file.getParentFile().toPath());
}
FileOutputStream stream = new FileOutputStream(file);
stream.write(content);
src/main/java/de/bommels05/befatorweb/links/LinkDatabase.java
+14 -17
View File
@@ -1,6 +1,7 @@
package de.bommels05.befatorweb.links;
import de.bommels05.befatorweb.BefatorRewriter;
import de.bommels05.befatorweb.BefatorWeb;
import de.bommels05.befatorweb.links.calc.BefatorDistanceCalculator;
import de.bommels05.befatorweb.links.calc.DistanceCalculator;
import de.bommels05.befatorweb.links.calc.V2DistanceCalculator;
@@ -33,33 +34,29 @@ public class LinkDatabase {
public static void addLink(String source, String destination) {
if (destination.startsWith("#")) {
return;
} else if (destination.substring(destination.indexOf("/") + 1).contains("#")) {
destination = destination.substring(0, destination.lastIndexOf("#"));
}
if (source.startsWith("/proxy/")) {
source = source.replaceFirst("/proxy/", "");
}
if (destination.startsWith("/proxy/")) {
destination = destination.replaceFirst("/proxy/", "https://");
if (destination.startsWith("http://")) {
destination = destination.replaceFirst("http://", "");
}
try {
URI base = new URI(source);
URI relative = new URI(destination);
URI result = base.resolve(relative);
String finalDestination = result.toString();
if (finalDestination.startsWith("https://")) {
//try {
/*if (finalDestination.startsWith("https://")) {
finalDestination = finalDestination.replaceFirst("https://", "");
}
}*/
if (source.equals(finalDestination) || hasDirectLink(source, finalDestination)) {
if (source.equals(destination) || hasDirectLink(source, destination)) {
return;
}
new LinkDBEntry(LINKS_TABLE, source, finalDestination).save();
} catch (URISyntaxException e) {
LinkDBEntry link = new LinkDBEntry(LINKS_TABLE, source, destination);
link.save();
BefatorWeb.addLinkToGraph(link);
/*} catch (URISyntaxException e) {
LOGGER.error("Invalid Link URL", e);
}
}*/
}
private static boolean hasDirectLink(String source, String destination) {