still trying to get routes and stuff figure out. Want to add a part of the tool to do a crawl on a website and figure out all of the routes, and make a map of them depending on the domain used.

.gitignore

@@ -37,7 +37,7 @@ inc/**
 
 # everything in the config directory
 config/routemaps/*
-
+config/pagemod/*
 # cookies.json file
 cookies.json
 

cmd/route.go

@@ -27,25 +27,15 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
-	"log"
-	"os"
 
 	httpserver "git.aetherial.dev/aeth/http-proxy/pkg"
 )
 
 func main() {
 
-	rmaps := httpserver.ReadRouteMap("/home/svcs/http-proxy/config/routemaps/static.semrush.com.json")
-	fmt.Printf("%v\n", rmaps)
-	b, err := json.Marshal(rmaps)
-	if err != nil {
-		log.Fatal(err)
+	rmaps := httpserver.LoadPageMods("./config/pagemod/rewrite.json")
+	for i := range rmaps.Content {
+		fmt.Printf("%+v\n", rmaps.Content[i])
 	}
-	err = os.WriteFile("./map.json", b, os.ModePerm)
-	if err != nil {
-		log.Fatal(err)
-	}
-
 }

pkg/client.go

@@ -27,6 +27,7 @@
 package httpserver
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"net/http"
@@ -41,17 +42,13 @@ Retrieve the site audit config file from Semrush
 	returns a byte array of the body, the content type of the resp, and an error
 */
 func (c *Controller) RetrieveStaticResource(method string, path string, ctx *gin.Context) ([]byte, string, int, error) {
-	cacheResp := c.GetResource(path)
-	if cacheResp != nil {
-		return cacheResp.Data, cacheResp.Ctype, cacheResp.Rcode, nil
-	}
 	url := fmt.Sprintf("%s%s", c.Config.FullAltAllowedDomain, path)
 	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
 		return nil, "", 500, err
 	}
 	c.setHeaders(req, ctx)
-
+	fmt.Printf("%+v\n", url)
 	resp, err := c.Client.Do(req)
 	if err != nil {
 		return nil, "", 500, err
@@ -61,10 +58,11 @@ func (c *Controller) RetrieveStaticResource(method string, path string, ctx *gin
 	if err != nil {
 		return nil, "", 500, err
 	}
+	altPage := c.pageMod(b)
 	if resp.StatusCode == 200 {
-		c.CacheResource(path, NewCachedResource(b, resp.Header.Get("content-type"), resp.StatusCode))
+		c.CacheResource(path, NewCachedResource(altPage, resp.Header.Get("content-type"), resp.StatusCode))
 	}
-	return c.pageMod(b), resp.Header.Get("content-type"), resp.StatusCode, nil
+	return altPage, resp.Header.Get("content-type"), resp.StatusCode, nil
 
 }
 
@@ -94,7 +92,8 @@ func (c *Controller) SiteauditApiCall(method string, path string, query string,
 	if err != nil {
 		return nil, "", 500, err
 	}
-	return c.pageMod(b), resp.Header.Get("content-type"), resp.StatusCode, nil
+	//	altPage := c.pageMod(b)
+	return b, resp.Header.Get("content-type"), resp.StatusCode, nil
 
 }
 
@@ -112,12 +111,7 @@ func (c *Controller) SemrushGeneric(ctx *gin.Context) ([]byte, string, int, erro
 	} else {
 		reqUrl = fmt.Sprintf("%s%s", c.Config.FullDomain, path)
 	}
-	cacheResp := c.GetResource(path)
-
-	if cacheResp != nil {
-		return cacheResp.Data, cacheResp.Ctype, cacheResp.Rcode, nil
-	}
-
+	fmt.Printf("Request URL: %s\n", reqUrl)
 	req, err := http.NewRequest(method, reqUrl, body)
 	if err != nil {
 		return nil, "", 500, err
@@ -139,15 +133,16 @@ func (c *Controller) SemrushGeneric(ctx *gin.Context) ([]byte, string, int, erro
 			ctx.Header(k, v[0])
 		}
 	}
+	altPage := c.pageMod(b)
 	if resp.StatusCode == 200 {
 		if query == "" {
 			if method == "GET" {
-				c.CacheResource(path, NewCachedResource(b, resp.Header.Get("content-type"), resp.StatusCode))
+				c.CacheResource(path, NewCachedResource(altPage, resp.Header.Get("content-type"), resp.StatusCode))
 			}
 		}
 	}
 
-	return c.pageMod(b), resp.Header.Get("content-type"), resp.StatusCode, nil
+	return altPage, resp.Header.Get("content-type"), resp.StatusCode, nil
 
 }
 
@@ -177,12 +172,12 @@ func (c *Controller) setHeaders(req *http.Request, ctx *gin.Context) {
 Rewrite all occurences of these values into the response body
 */
 func (c *Controller) pageMod(data []byte) []byte {
-	newBody := strings.ReplaceAll(string(data), "\"srf-browser-unhappy\"", "\"srf-browser-unhappy\" style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, "\"srf-navbar__right\"", "\"srf-navbar__right\" style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, "<footer", "<footer style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, "\"srf-report-sidebar-extras\"", "\"srf-report-sidebar-extra\" style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, c.Config.AllowedDomain, c.Config.ProxyAddr)
-	newBody = strings.ReplaceAll(newBody, c.Config.AltAllowedDomain, c.Config.ProxyAddr)
-	return []byte(newBody)
+	for idx := range c.PageMods.Content {
+		data = bytes.ReplaceAll(data, []byte(c.PageMods.Content[idx].Search), []byte(c.PageMods.Content[idx].Sub))
+	}
+	data = bytes.ReplaceAll(data, []byte(c.Config.AllowedDomain), []byte(c.Config.ProxyAddr))
+	data = bytes.ReplaceAll(data, []byte(c.Config.AltAllowedDomain), []byte(c.Config.ProxyAddr))
+
+	return data
 
 }

pkg/controller.go

@@ -52,6 +52,7 @@ var NonmutableHeaders = map[string]struct{}{
 type Controller struct {
 	Config    *HttpServerConfig
 	RouteMaps map[string]*RouteMapping
+	PageMods  *AllPageMods
 	Client    *http.Client
 	SiteUrl   *url.URL
 	cache     *cache.Cache
@@ -78,10 +79,11 @@ func NewController(cfg *HttpServerConfig, routeMapping map[string]*RouteMapping)
 	if err != nil {
 		log.Fatal(err)
 	}
+	pgMod := LoadPageMods(cfg.PageModPath)
 	jar.SetCookies(domain, sessCookies)
 	cache := cache.New(24*time.Hour, 10*time.Minute)
 
-	return &Controller{Config: cfg, Client: &http.Client{Jar: jar}, SiteUrl: domain, cache: cache, RouteMaps: routeMapping}
+	return &Controller{Config: cfg, Client: &http.Client{Jar: jar}, SiteUrl: domain, cache: cache, RouteMaps: routeMapping, PageMods: pgMod}
 }
 
 /*
@@ -90,20 +92,21 @@ This handler will be responsible for proxying out the GET requests that the serv
 func (c *Controller) Get(ctx *gin.Context) {
 	incomingPath := ctx.Param("ProxiedPath")
 	routeSplit := strings.Split(incomingPath, "/")
-	if len(routeSplit) > 1 {
-		baseRoute := strings.Join(routeSplit[:len(routeSplit)-1], "/")
-
-		_, ok := c.RouteMaps[c.Config.AltAllowedDomain].RouteSet[baseRoute]
-		if ok {
-			data, ctype, rcode, err := c.RetrieveStaticResource(ctx.Request.Method, incomingPath, ctx)
-			if err != nil {
-				log.Fatal(err, " failed to reroute to the static domain")
-			}
-			ctx.Data(rcode, ctype, data)
-			return
-		}
+	if incomingPath == "/" {
+		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/overview/")
+	}
+	if incomingPath == "/_compatibility/traffic/overview/" {
+		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/traffic/overview/ebay.com")
+	}
+	cacheHit := c.GetResource(incomingPath)
+	if cacheHit != nil {
+		ctx.Data(cacheHit.Rcode, cacheHit.Ctype, cacheHit.Data)
+		return
 	}
-	_, ok := c.RouteMaps[c.Config.AltAllowedDomain].RouteSet[incomingPath]
+
+	lastElem := routeSplit[len(routeSplit)-1]
+	baseRoute := strings.TrimSuffix(ctx.Param("ProxiedPath"), lastElem)
+	_, ok := c.RouteMaps[c.Config.AltAllowedDomain].RouteSet[baseRoute]
 	if ok {
 		data, ctype, rcode, err := c.RetrieveStaticResource(ctx.Request.Method, incomingPath, ctx)
 		if err != nil {
@@ -128,20 +131,6 @@ func (c *Controller) Get(ctx *gin.Context) {
 		})
 		return
 	}
+	ctx.Data(rcode, ctype, data)
 
-	if incomingPath == "/" {
-		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/overview/")
-	}
-	if incomingPath == "/_compatibility/traffic/overview/" {
-		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/traffic/overview/ebay.com")
-	}
-
-	// TODO: this should also honestly be a configuration thing, so that i can extend page modifications
-	newBody := strings.ReplaceAll(string(data), "\"srf-browser-unhappy\"", "\"srf-browser-unhappy\" style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, "\"srf-navbar__right\"", "\"srf-navbar__right\" style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, "<footer", "<footer style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, "\"srf-report-sidebar-extras\"", "\"srf-report-sidebar-extra\" style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, c.Config.AllowedDomain, c.Config.ProxyAddr)
-	newBody = strings.ReplaceAll(newBody, c.Config.AltAllowedDomain, c.Config.ProxyAddr)
-	ctx.Data(rcode, ctype, []byte(newBody))
 }

pkg/include.go

@@ -48,6 +48,7 @@ type HttpServerConfig struct {
 	UseSsl               bool   `json:"use_ssl"`
 	ProxyAddr            string `json:"proxy_addr"`
 	RouteMapDir          string `json:"routemap_dir"`
+	PageModPath          string `json:"page_mod_path"`
 	FullProxyDomain      string // the domain name of the proxied site with the protocol
 	CookieJar            []*http.Cookie
 	PhpSession           *http.Cookie

pkg/pagemod.go

@@ -0,0 +1,62 @@
+/*
+				GNU GENERAL PUBLIC LICENSE
+                 Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ http-wokou, An HTTP Proxying framework for bypassing DNS Security
+ Copyright (C) 2024 Russell Hrubesky, ChiralWorks Software LLC
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+*/
+
+package httpserver
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+)
+
+type PageMod struct {
+	Target string `json:"target"`
+	Search string `json:"search"`
+	Sub    string `json:"sub"`
+}
+
+type AllPageMods struct {
+	Content []*PageMod `json:"modifications"`
+}
+
+/*
+Load page modifications from the config/pagemod directory
+
+	:param loc: the path to the pagemods directory
+*/
+func LoadPageMods(loc string) *AllPageMods {
+	b, err := os.ReadFile(loc)
+	if err != nil {
+		log.Fatal("Couldnt load the page modification config from: ", loc, ". Error: ", err)
+	}
+	var pgMod AllPageMods
+	err = json.Unmarshal(b, &pgMod)
+	if err != nil {
+		log.Fatal("There was an error loading the configuration: ", err)
+	}
+	return &pgMod
+
+}