made config changes

cmd/http-wokou.go

@@ -49,9 +49,9 @@ func main() {
 		"https://sem.bunnytool.shop",
 	}
 	e.Use(cors.New(config))
-	rmaps := httpserver.PopulateRouteMaps(cfg.RouteMapDir)
-	fmt.Printf("%v\n", rmaps["static.semrush.com"])
+	rmaps := httpserver.ReadRouteMap(cfg.RouteMapPath)
 	httpserver.RegisterRoutes(e, cfg, rmaps)
+
 	e.RunTLS(fmt.Sprintf("%s:%v", "0.0.0.0", cfg.HttpsPort), "/etc/letsencrypt/live/void-society.online/fullchain.pem", "/etc/letsencrypt/live/void-society.online/privkey.pem")
 
 }

cmd/route.go

@@ -27,15 +27,39 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
 	httpserver "git.aetherial.dev/aeth/http-proxy/pkg"
 )
 
+type Test struct {
+	Data *httpserver.AllPageMods
+}
+
 func main() {
+	var maps Test
+	maps.Data = httpserver.LoadPageMods("./config/pagemod/rewrite.json")
+	c := make(chan os.Signal)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func(ds Test) {
+		b, err := json.Marshal(ds)
+		if err != nil {
+			log.Fatal("failed to marshal struct: ", err)
+		}
+		<-c
+		os.WriteFile("MadeItOutTheJug.json", b, os.ModePerm)
+		os.Exit(1)
+	}(maps)
 
-	rmaps := httpserver.LoadPageMods("./config/pagemod/rewrite.json")
-	for i := range rmaps.Content {
-		fmt.Printf("%+v\n", rmaps.Content[i])
+	for {
+		fmt.Println("sleeping...")
+		time.Sleep(15 * time.Second) // or runtime.Gosched() or similar per @misterbee
 	}
+
 }

pkg/cache.go

@@ -28,25 +28,28 @@ package httpserver
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/patrickmn/go-cache"
 )
 
 type CachedResource struct {
-	Data  []byte
-	Ctype string
-	Rcode int
+	Data    []byte
+	Headers *http.Header
+	Ctype   string
+	Rcode   int
 }
 
-func NewCachedResource(data []byte, ctype string, rcode int) *CachedResource {
+func NewCachedResource(data []byte, headers *http.Header, rcode int) *CachedResource {
 	return &CachedResource{
-		Data:  data,
-		Ctype: ctype,
-		Rcode: rcode,
+		Data:    data,
+		Headers: headers,
+		Rcode:   rcode,
 	}
 }
 
 func (c *Controller) CacheResource(key string, resource *CachedResource) {
+	fmt.Printf("Cached resource for: %s\n", key)
 	c.cache.Set(key, resource, cache.DefaultExpiration)
 }
 

pkg/client.go

@@ -30,119 +30,63 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"strings"
-
-	"github.com/gin-gonic/gin"
+	"sync"
 )
 
-/*
-Retrieve the site audit config file from Semrush
-
-	returns a byte array of the body, the content type of the resp, and an error
-*/
-func (c *Controller) RetrieveStaticResource(method string, path string, ctx *gin.Context) ([]byte, string, int, error) {
-	url := fmt.Sprintf("%s%s", c.Config.FullAltAllowedDomain, path)
-	req, err := http.NewRequest(method, url, nil)
-	if err != nil {
-		return nil, "", 500, err
-	}
-	c.setHeaders(req, ctx)
-	fmt.Printf("%+v\n", url)
-	resp, err := c.Client.Do(req)
-	if err != nil {
-		return nil, "", 500, err
-	}
-	defer resp.Body.Close()
-	b, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, "", 500, err
-	}
-	altPage := c.pageMod(b)
-	if resp.StatusCode == 200 {
-		c.CacheResource(path, NewCachedResource(altPage, resp.Header.Get("content-type"), resp.StatusCode))
-	}
-	return altPage, resp.Header.Get("content-type"), resp.StatusCode, nil
-
-}
-
-/*
-Perform a call against the siteaudit api
-
-	:param path: the URI path with the query
-	:param query: the query to add to the request
-	:param body: an io.Reader to push into the request body
-	:returns a byte array of the response, the content type and an error
-*/
-func (c *Controller) SiteauditApiCall(method string, path string, query string, body io.Reader, ctx *gin.Context) ([]byte, string, int, error) {
-
-	query = strings.ReplaceAll(query, c.Config.FullProxyDomain, c.Config.FullDomain)
-	url := fmt.Sprintf("%s%s?%s", c.Config.FullDomain, path, query)
-	req, err := http.NewRequest(method, url, body)
-	if err != nil {
-		return nil, "", 500, err
-	}
-	c.setHeaders(req, ctx)
-	resp, err := c.Client.Do(req)
-	if err != nil {
-		return nil, "", 500, err
-	}
-	defer resp.Body.Close()
-	b, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, "", 500, err
-	}
-	//	altPage := c.pageMod(b)
-	return b, resp.Header.Get("content-type"), resp.StatusCode, nil
-
-}
-
 /*
 Generic site call to the semrush site
 */
-func (c *Controller) SemrushGeneric(ctx *gin.Context) ([]byte, string, int, error) {
-	path := ctx.Param("ProxiedPath")
-	method := ctx.Request.Method
-	query := ctx.Request.URL.RawQuery
-	body := ctx.Request.Body
-	var reqUrl string
-	if query != "" {
-		reqUrl = fmt.Sprintf("%s%s?%s", c.Config.FullDomain, path, query)
-	} else {
-		reqUrl = fmt.Sprintf("%s%s", c.Config.FullDomain, path)
-	}
-	fmt.Printf("Request URL: %s\n", reqUrl)
+func (c *Controller) RequestGeneric(method string, host string, path string, hdrs *http.Header, body io.Reader) ([]byte, *http.Header, int, error) {
+	reqUrl := fmt.Sprintf("https://%s%s", host, path)
+
 	req, err := http.NewRequest(method, reqUrl, body)
 	if err != nil {
-		return nil, "", 500, err
+		return nil, nil, 500, err
 	}
-	c.setHeaders(req, ctx)
+	c.setHeaders(req, hdrs)
 	resp, err := c.Client.Do(req)
 	if err != nil {
-		return nil, "", 500, err
+		return nil, nil, 500, err
 	}
 	defer resp.Body.Close()
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, "", 500, err
+		return nil, nil, 500, err
 	}
 
-	for k, v := range resp.Header {
-		_, ok := NonmutableHeaders[k]
-		if !ok {
-			ctx.Header(k, v[0])
-		}
-	}
 	altPage := c.pageMod(b)
-	if resp.StatusCode == 200 {
-		if query == "" {
+	if !strings.Contains(path, "?") {
+		if resp.StatusCode == 200 {
 			if method == "GET" {
-				c.CacheResource(path, NewCachedResource(altPage, resp.Header.Get("content-type"), resp.StatusCode))
+				c.CacheResource(path, NewCachedResource(altPage, &resp.Header, resp.StatusCode))
 			}
 		}
 	}
 
-	return altPage, resp.Header.Get("content-type"), resp.StatusCode, nil
+	return altPage, &resp.Header, resp.StatusCode, nil
+
+}
+
+func (c *Controller) TryHosts(method string, path string, hdrs *http.Header, body io.Reader, hosts []string) {
+	var wg sync.WaitGroup
+	for idx := range hosts {
+		wg.Add(1)
+		go func(method string, host string, path string, hdrs *http.Header, body io.Reader) {
+			defer wg.Done()
+			_, _, rcode, err := c.RequestGeneric(method, host, path, hdrs, body)
+			if err != nil {
+				log.Fatal("Fatal Error creating request in a RequestGeneric method: ", err)
+			}
+			if rcode == 200 {
+				basePath := strings.Split(path, "?")[0]
+				c.RouteMaps.MapUriToDomain(basePath, host)
+			}
+		}(method, hosts[idx], path, hdrs, body)
+	}
+	wg.Wait()
 
 }
 
@@ -151,15 +95,14 @@ Sets the request headers to whatever is defined in this private method
 
 	:param req: a pointer to an HTTP request
 */
-func (c *Controller) setHeaders(req *http.Request, ctx *gin.Context) {
+func (c *Controller) setHeaders(req *http.Request, hdrs *http.Header) {
 
 	req.AddCookie(c.Config.PhpSession)
 	req.AddCookie(c.Config.SsoToken)
 	req.Header.Set("User-Agent", c.Config.UserAgent)
 	req.Header.Set("Referer", c.Config.FullDomain)
 	req.Header.Set("Origin", c.Config.FullDomain)
-
-	for k, v := range ctx.Request.Header {
+	for k, v := range *hdrs {
 		_, ok := NonmutableHeaders[k]
 		if !ok {
 			req.Header.Add(k, v[0])
@@ -177,7 +120,7 @@ func (c *Controller) pageMod(data []byte) []byte {
 	}
 	data = bytes.ReplaceAll(data, []byte(c.Config.AllowedDomain), []byte(c.Config.ProxyAddr))
 	data = bytes.ReplaceAll(data, []byte(c.Config.AltAllowedDomain), []byte(c.Config.ProxyAddr))
-
+	data = bytes.ReplaceAll(data, []byte("api-iam.intercom.io"), []byte(c.Config.ProxyAddr))
 	return data
 
 }

pkg/controller.go

@@ -31,7 +31,6 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
-	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -51,7 +50,7 @@ var NonmutableHeaders = map[string]struct{}{
 
 type Controller struct {
 	Config    *HttpServerConfig
-	RouteMaps map[string]*RouteMapping
+	RouteMaps *RouteMap
 	PageMods  *AllPageMods
 	Client    *http.Client
 	SiteUrl   *url.URL
@@ -67,7 +66,7 @@ Returns a new Controller struct to register routes to the gin router
 
 	:param cfg: A pointer to an HttpServerConfig struct
 */
-func NewController(cfg *HttpServerConfig, routeMapping map[string]*RouteMapping) *Controller {
+func NewController(cfg *HttpServerConfig, routeMap *RouteMap) *Controller {
 
 	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
 	if err != nil {
@@ -83,7 +82,9 @@ func NewController(cfg *HttpServerConfig, routeMapping map[string]*RouteMapping)
 	jar.SetCookies(domain, sessCookies)
 	cache := cache.New(24*time.Hour, 10*time.Minute)
 
-	return &Controller{Config: cfg, Client: &http.Client{Jar: jar}, SiteUrl: domain, cache: cache, RouteMaps: routeMapping, PageMods: pgMod}
+	return &Controller{Config: cfg, Client: &http.Client{Jar: jar, CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}}, SiteUrl: domain, cache: cache, RouteMaps: routeMap, PageMods: pgMod}
 }
 
 /*
@@ -91,7 +92,6 @@ This handler will be responsible for proxying out the GET requests that the serv
 */
 func (c *Controller) Get(ctx *gin.Context) {
 	incomingPath := ctx.Param("ProxiedPath")
-	routeSplit := strings.Split(incomingPath, "/")
 	if incomingPath == "/" {
 		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/overview/")
 	}
@@ -100,37 +100,48 @@ func (c *Controller) Get(ctx *gin.Context) {
 	}
 	cacheHit := c.GetResource(incomingPath)
 	if cacheHit != nil {
-		ctx.Data(cacheHit.Rcode, cacheHit.Ctype, cacheHit.Data)
+		for k, v := range *cacheHit.Headers {
+			_, ok := NonmutableHeaders[k]
+			if !ok {
+				for i := range v {
+					ctx.Header(k, v[i])
+				}
+			}
+		}
+		ctx.Data(cacheHit.Rcode, cacheHit.Headers.Get("content-type"), cacheHit.Data)
 		return
 	}
 
-	lastElem := routeSplit[len(routeSplit)-1]
-	baseRoute := strings.TrimSuffix(ctx.Param("ProxiedPath"), lastElem)
-	_, ok := c.RouteMaps[c.Config.AltAllowedDomain].RouteSet[baseRoute]
-	if ok {
-		data, ctype, rcode, err := c.RetrieveStaticResource(ctx.Request.Method, incomingPath, ctx)
+	dname, ok := c.RouteMaps.GetMappedDomain(incomingPath)
+	if ok { // below, RequestURI() returns the whole URI with the query
+		/*
+			if ctx.Request.Body != nil {
+				data, err := io.ReadAll(ctx.Request.Body)
+				if err != nil {
+					log.Fatal("error reading byte array: ", err)
+				}
+
+				data = bytes.ReplaceAll(data, []byte(c.Config.ProxyAddr), []byte("api-iam.intercom.io"))
+				ctx.Request.Body = io.NopCloser(bytes.NewReader(data))
+
+			}
+		*/
+		data, headers, rcode, err := c.RequestGeneric(ctx.Request.Method, dname, ctx.Request.URL.RequestURI(), &ctx.Request.Header, ctx.Request.Body)
 		if err != nil {
-			log.Fatal(err, " failed to reroute to the static domain")
+			log.Fatal(err, " failed to route the request: ", incomingPath, " to the target domain: ", dname, " Error: ", err)
 		}
-		ctx.Data(rcode, ctype, data)
-		return
-	}
-	_, ok = c.RouteMaps[c.Config.AllowedDomain].RouteSet[incomingPath]
-	if ok {
-		data, ctype, rcode, err := c.SiteauditApiCall(ctx.Request.Method, incomingPath, ctx.Request.URL.RawQuery, ctx.Request.Body, ctx)
-		if err != nil {
-			log.Fatal(err, " failed to route to the root domain")
+		for k, v := range *headers {
+			_, ok := NonmutableHeaders[k]
+			if !ok {
+				ctx.Header(k, v[0])
+			}
 		}
-		ctx.Data(rcode, ctype, data)
+		ctx.Header("access-control-allow-origin", c.Config.FullProxyDomain)
+		ctx.Data(rcode, headers.Get("content-type"), data)
+		//	c.RouteMaps.ExportRouteMap()
 		return
 	}
-	data, ctype, rcode, err := c.SemrushGeneric(ctx)
-	if err != nil {
-		ctx.JSON(rcode, map[string]string{
-			"Error": err.Error(),
-		})
-		return
-	}
-	ctx.Data(rcode, ctype, data)
+
+	c.TryHosts(ctx.Request.Method, ctx.Request.URL.RequestURI(), &ctx.Request.Header, ctx.Request.Body, c.Config.KnownHosts)
 
 }

pkg/include.go

@@ -29,27 +29,30 @@ package httpserver
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
-	"path"
+	"strings"
+	"time"
+
+	"github.com/patrickmn/go-cache"
 )
 
 type HttpServerConfig struct {
-	HttpPort             int    `json:"http_port"`
-	HttpsPort            int    `json:"https_port"`
-	AllowedDomain        string `json:"allowed_domain"`
-	FullDomain           string // The domain name with the protocol before it
-	AltAllowedDomain     string `json:"alt_allowed_domain"` // alternate domain that resources are sourced from
-	FullAltAllowedDomain string // the alt domain with the protocol
-	Proto                string `json:"proto"` // http/https
-	UserAgent            string `json:"user_agent"`
-	UseSsl               bool   `json:"use_ssl"`
-	ProxyAddr            string `json:"proxy_addr"`
-	RouteMapDir          string `json:"routemap_dir"`
-	PageModPath          string `json:"page_mod_path"`
-	FullProxyDomain      string // the domain name of the proxied site with the protocol
+	HttpPort             int      `json:"http_port"`
+	HttpsPort            int      `json:"https_port"`
+	AllowedDomain        string   `json:"allowed_domain"`
+	FullDomain           string   // The domain name with the protocol before it
+	AltAllowedDomain     string   `json:"alt_allowed_domain"` // alternate domain that resources are sourced from
+	FullAltAllowedDomain string   // the alt domain with the protocol
+	Proto                string   `json:"proto"` // http/https
+	UserAgent            string   `json:"user_agent"`
+	UseSsl               bool     `json:"use_ssl"`
+	ProxyAddr            string   `json:"proxy_addr"`
+	RouteMapPath         string   `json:"route_map_path"`
+	PageModPath          string   `json:"page_mod_path"`
+	FullProxyDomain      string   // the domain name of the proxied site with the protocol
+	KnownHosts           []string `json:"known_hosts"`
 	CookieJar            []*http.Cookie
 	PhpSession           *http.Cookie
 	SsoToken             *http.Cookie
@@ -71,6 +74,73 @@ type RouteMapping struct {
 	RouteSet   map[string]struct{}
 }
 
+type RouteMap struct {
+	Mappings map[string]string `json:"mappings"`
+	Shotgun  map[string]string `json:"shotgun"`
+	MapCache *cache.Cache
+}
+
+type RouteMapper interface {
+	mapUriToDomain(string, string)
+	GetMappedDomain(string) (string, bool)
+	ExportRouteMaps()
+}
+
+/*
+Set a route to exist for the URI to the specific domain
+
+	:param uri: the URI to set the route for
+	:param domain: the domain name to resolve the uri to
+*/
+func (r *RouteMap) MapUriToDomain(uri string, domain string) {
+	r.MapCache.Set(uri, domain, cache.DefaultExpiration)
+
+}
+
+// returns the domain/url that the uri belongs to as defined in the routemap
+func (r *RouteMap) GetMappedDomain(uri string) (string, bool) {
+	dname, ok := r.MapCache.Get(uri)
+	if ok {
+		return fmt.Sprint(dname), true
+
+	}
+
+	for k, v := range r.Shotgun {
+
+		if strings.Contains(uri, k) {
+			return v, true
+		}
+	}
+
+	return "", false
+}
+
+// This populates the cache in a RouteMap with the data from the config file
+func (r *RouteMap) populateRouteMaps() {
+	for k, v := range r.Mappings {
+		r.MapUriToDomain(k, v)
+	}
+}
+
+// Exports the cache into a JSON-friendly data structure (so that it can be written to the file system)
+func (r *RouteMap) ExportRouteMap() {
+	routeMapOut := &RouteMap{
+		Mappings: map[string]string{},
+	}
+
+	cachedRoutes := r.MapCache.Items()
+	for k, v := range cachedRoutes {
+		routeMapOut.Mappings[k] = fmt.Sprint(v.Object)
+	}
+
+	b, err := json.Marshal(routeMapOut)
+	if err != nil {
+		log.Fatal("failed to marshal struct: ", err)
+	}
+	os.WriteFile("./MadeItOutTheJug.json", b, os.ModePerm)
+
+}
+
 /*
 Reads the server configuration file, along with the cookie file so that the correlated account can be
 accessed through the proxy
@@ -119,42 +189,22 @@ func ReadConfig(loc string) (*HttpServerConfig, error) {
 
 	}
 
-	fmt.Printf("%+v\n", cfg)
 	return &cfg, err
 
 }
 
-func ReadRouteMap(loc string) *RouteMapping {
+func ReadRouteMap(loc string) *RouteMap {
 	f, err := os.ReadFile(loc)
 	if err != nil {
 		log.Fatal(err)
 	}
-	var mapfile RouteMapping
+	var mapfile RouteMap
 	err = json.Unmarshal(f, &mapfile)
 	if err != nil {
 		log.Fatal(err)
 	}
-	rmap := map[string]struct{}{}
-	for idx := range mapfile.UriPaths {
-		rmap[mapfile.UriPaths[idx]] = struct{}{}
-	}
-	mapfile.RouteSet = rmap
-
+	mapfile.MapCache = cache.New(24*time.Hour, 10*time.Minute)
+	mapfile.populateRouteMaps()
 	return &mapfile
 
 }
-
-func PopulateRouteMaps(dir string) map[string]*RouteMapping {
-
-	routeMaps := map[string]*RouteMapping{}
-	fnames, err := ioutil.ReadDir(dir)
-	if err != nil {
-		log.Fatal("Failed to read the routemap directory: ", err)
-	}
-	for _, f := range fnames {
-		mapping := ReadRouteMap(path.Join(dir, f.Name()))
-		routeMaps[f.Name()] = mapping
-
-	}
-	return routeMaps
-}

pkg/routing.go

@@ -27,6 +27,10 @@
 package httpserver
 
 import (
+	"os"
+	"os/signal"
+	"syscall"
+
 	"github.com/gin-gonic/gin"
 )
 
@@ -36,8 +40,15 @@ Registers the exposed route to utilize the handler function
 	:param e: pointer to a gin.Engine struct
 	:param cfg: pointer to an HttpServerConfig struct
 */
-func RegisterRoutes(e *gin.Engine, cfg *HttpServerConfig, rmaps map[string]*RouteMapping) {
+func RegisterRoutes(e *gin.Engine, cfg *HttpServerConfig, rmaps *RouteMap) {
 	c := NewController(cfg, rmaps)
+	sigChanel := make(chan os.Signal)
+	signal.Notify(sigChanel, os.Interrupt, syscall.SIGINT)
+	go func(core *Controller) {
+		<-sigChanel
+		c.RouteMaps.ExportRouteMap()
+		os.Exit(1)
+	}(c)
 	web := e.Group("")
 	web.Any("/*ProxiedPath", c.Get)