still working on optimizing, added caching as well as made it easy to extend the routes that are proxied to seperate domains

.gitignore

@@ -35,6 +35,9 @@ inc/**
 # ignoring real config files
 .config.*
 
+# everything in the config directory
+config/**
+
 # cookies.json file
 cookies.json
 

cmd/route.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	httpserver "git.aetherial.dev/aeth/http-proxy/pkg"
+)
+
+func main() {
+
+	rmaps := httpserver.ReadRouteMap("/home/svcs/http-proxy/config/routemaps/static.semrush.com.json")
+	fmt.Printf("%v\n", rmaps)
+	b, err := json.Marshal(rmaps)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = os.WriteFile("./map.json", b, os.ModePerm)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}

cmd/server.go

@@ -14,7 +14,7 @@ func main() {
 	if err != nil {
 		log.Fatal("Couldnt read config: ", err)
 	}
-	fmt.Printf("%+v\n", cfg)
+
 	e := gin.Default()
 	config := cors.DefaultConfig()
 	config.AllowOrigins = []string{
@@ -23,8 +23,9 @@ func main() {
 		"https://sem.bunnytool.shop",
 	}
 	e.Use(cors.New(config))
-
-	httpserver.RegisterRoutes(e, cfg)
+	rmaps := httpserver.PopulateRouteMaps(cfg.RouteMapDir)
+	fmt.Printf("%v\n", rmaps["static.semrush.com"])
+	httpserver.RegisterRoutes(e, cfg, rmaps)
 	e.RunTLS(fmt.Sprintf("%s:%v", "0.0.0.0", cfg.HttpsPort), "/etc/letsencrypt/live/void-society.online/fullchain.pem", "/etc/letsencrypt/live/void-society.online/privkey.pem")
 
 }

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
 	github.com/pinecat/netcookiejar v0.3.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect

go.sum

@@ -58,6 +58,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
 github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=

pkg/cache.go

@@ -0,0 +1,34 @@
+package httpserver
+
+import (
+	"fmt"
+
+	"github.com/patrickmn/go-cache"
+)
+
+type CachedResource struct {
+	Data  []byte
+	Ctype string
+	Rcode int
+}
+
+func NewCachedResource(data []byte, ctype string, rcode int) *CachedResource {
+	return &CachedResource{
+		Data:  data,
+		Ctype: ctype,
+		Rcode: rcode,
+	}
+}
+
+func (c *Controller) CacheResource(key string, resource *CachedResource) {
+	c.cache.Set(key, resource, cache.DefaultExpiration)
+}
+
+func (c *Controller) GetResource(key string) *CachedResource {
+	resource, found := c.cache.Get(key)
+	if found {
+		fmt.Printf("Cache Hit! Found resource for URI: %s\n", key)
+		return resource.(*CachedResource)
+	}
+	return nil
+}

pkg/client.go

@@ -15,6 +15,10 @@ Retrieve the site audit config file from Semrush
 	returns a byte array of the body, the content type of the resp, and an error
 */
 func (c *Controller) RetrieveStaticResource(method string, path string, ctx *gin.Context) ([]byte, string, int, error) {
+	cacheResp := c.GetResource(path)
+	if cacheResp != nil {
+		return cacheResp.Data, cacheResp.Ctype, cacheResp.Rcode, nil
+	}
 	url := fmt.Sprintf("%s%s", c.Config.FullAltAllowedDomain, path)
 	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
@@ -31,6 +35,9 @@ func (c *Controller) RetrieveStaticResource(method string, path string, ctx *gin
 	if err != nil {
 		return nil, "", 500, err
 	}
+	if resp.StatusCode == 200 {
+		c.CacheResource(path, NewCachedResource(b, resp.Header.Get("content-type"), resp.StatusCode))
+	}
 	return b, resp.Header.Get("content-type"), resp.StatusCode, nil
 
 }
@@ -43,34 +50,49 @@ Perform a call against the siteaudit api
 	:param body: an io.Reader to push into the request body
 	:returns a byte array of the response, the content type and an error
 */
-func (c *Controller) SiteauditApiCall(method string, path string, query string, body io.Reader, ctx *gin.Context) ([]byte, string, error) {
+func (c *Controller) SiteauditApiCall(method string, path string, query string, body io.Reader, ctx *gin.Context) ([]byte, string, int, error) {
+
 	query = strings.ReplaceAll(query, c.Config.FullProxyDomain, c.Config.FullDomain)
 	url := fmt.Sprintf("%s%s?%s", c.Config.FullDomain, path, query)
 	req, err := http.NewRequest(method, url, body)
 	if err != nil {
-		return nil, "", err
+		return nil, "", 500, err
 	}
 	c.setHeaders(req, ctx)
 	resp, err := c.Client.Do(req)
 	if err != nil {
-		return nil, "", err
+		return nil, "", 500, err
 	}
 	defer resp.Body.Close()
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, "", err
+		return nil, "", 500, err
 	}
-	return b, resp.Header.Get("content-type"), nil
+	return b, resp.Header.Get("content-type"), resp.StatusCode, nil
 
 }
 
 /*
 Generic site call to the semrush site
 */
-func (c *Controller) SemrushGeneric(method string, path string, body io.Reader, ctx *gin.Context) ([]byte, string, int, error) {
+func (c *Controller) SemrushGeneric(ctx *gin.Context) ([]byte, string, int, error) {
+	path := ctx.Param("ProxiedPath")
+	method := ctx.Request.Method
+	query := ctx.Request.URL.RawQuery
+	body := ctx.Request.Body
+	var reqUrl string
+	if query != "" {
+		reqUrl = fmt.Sprintf("%s%s?%s", c.Config.FullDomain, path, query)
+	} else {
+		reqUrl = fmt.Sprintf("%s%s", c.Config.FullDomain, path)
+	}
+	cacheResp := c.GetResource(path)
 
-	url := fmt.Sprintf("%s%s", c.Config.FullDomain, path)
-	req, err := http.NewRequest(method, url, body)
+	if cacheResp != nil {
+		return cacheResp.Data, cacheResp.Ctype, cacheResp.Rcode, nil
+	}
+
+	req, err := http.NewRequest(method, reqUrl, body)
 	if err != nil {
 		return nil, "", 500, err
 	}
@@ -91,6 +113,13 @@ func (c *Controller) SemrushGeneric(method string, path string, body io.Reader,
 			ctx.Header(k, v[0])
 		}
 	}
+	if resp.StatusCode == 200 {
+		if query == "" {
+			if method == "GET" {
+				c.CacheResource(path, NewCachedResource(b, resp.Header.Get("content-type"), resp.StatusCode))
+			}
+		}
+	}
 
 	return b, resp.Header.Get("content-type"), resp.StatusCode, nil
 

pkg/controller.go

@@ -6,39 +6,44 @@ import (
 	"net/http/cookiejar"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/patrickmn/go-cache"
 	"golang.org/x/net/publicsuffix"
 )
 
 // TODO: does tihs need to be a configuration thing? How would i handle doing rewrite rules?
-var staticRoutes = [...]string{
-	"/siteaudit/i18n/messages_en",
-	"/siteaudit/index/",
-	"/siteaudit/review/",
-	"/seo-dashboard/release/",
-	"/competitive-list-widget/",
-	"/backlink-audit/landing/",
-	"/link-building-tool/landing/",
-	"/keyword-overview/",
-	"/keyword-gap/",
-	"/oti/prod/organic_traffic_insights/",
-	"/oti/prod/organic-traffic-insights",
-	"/ajst/",
-	"/listing-management/landings/",
-	"/listing-management/landing-reviews/",
-	"/messaging/apps/",
-	"/domain-overview/",
-	"/traffic-analytics/",
-	"/organic-research/",
-	"/keyword-magic/kmt_",
-	"/keyword-manager-assets/",
-	"/position-tracking/landing/",
+// TODO: make these routes cached on the proxy, so that when a client requests them, they arent being tunneled through to the actual site
+var staticRoutes = map[string]struct{}{
+	"/siteaudit/i18n":                     struct{}{},
+	"/siteaudit/index":                    struct{}{},
+	"/siteaudit/review":                   struct{}{},
+	"/seo-dashboard/release":              struct{}{},
+	"/competitive-list-widget":            struct{}{},
+	"/backlink-audit/landing":             struct{}{},
+	"/link-building-tool/landing":         struct{}{},
+	"/keyword-overview":                   struct{}{},
+	"/keyword-gap":                        struct{}{},
+	"/oti/prod/organic_traffic_insights":  struct{}{},
+	"/oti/prod/organic-traffic-insights":  struct{}{},
+	"/ajst":                               struct{}{},
+	"/listing-management/landings":        struct{}{},
+	"/listing-management/landing-reviews": struct{}{},
+	"/messaging/apps/":                    struct{}{},
+	"/domain-overview":                    struct{}{},
+	"/traffic-analytics":                  struct{}{},
+	"/organic-research":                   struct{}{},
+	"/keyword-magic/kmt_":                 struct{}{},
+	"/keyword-manager-assets":             struct{}{},
+	"/position-tracking/landing":          struct{}{},
 }
 
-var apiRoutes = [...]string{
-	"/projects/api/",
-	"/siteaudit/api/",
+var apiRoutes = map[string]struct{}{
+	"/siteaudit/api/campaigns/seolist": struct{}{},
+	"/siteaudit/api/system-status":     struct{}{},
+	"/siteaudit/api/limits":            struct{}{},
+	"/projects/api/limits":             struct{}{},
 }
 
 // Implementing a 'set'
@@ -52,9 +57,11 @@ var NonmutableHeaders = map[string]struct{}{
 }
 
 type Controller struct {
-	Config  *HttpServerConfig
-	Client  *http.Client
-	SiteUrl *url.URL
+	Config    *HttpServerConfig
+	RouteMaps map[string]*RouteMapping
+	Client    *http.Client
+	SiteUrl   *url.URL
+	cache     *cache.Cache
 }
 
 type ProxyCookies struct {
@@ -66,7 +73,8 @@ Returns a new Controller struct to register routes to the gin router
 
 	:param cfg: A pointer to an HttpServerConfig struct
 */
-func NewController(cfg *HttpServerConfig) *Controller {
+func NewController(cfg *HttpServerConfig, routeMapping map[string]*RouteMapping) *Controller {
+
 	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
 	if err != nil {
 		log.Fatal(err)
@@ -78,8 +86,9 @@ func NewController(cfg *HttpServerConfig) *Controller {
 		log.Fatal(err)
 	}
 	jar.SetCookies(domain, sessCookies)
+	cache := cache.New(24*time.Hour, 10*time.Minute)
 
-	return &Controller{Config: cfg, Client: &http.Client{Jar: jar}, SiteUrl: domain}
+	return &Controller{Config: cfg, Client: &http.Client{Jar: jar}, SiteUrl: domain, cache: cache, RouteMaps: routeMapping}
 }
 
 /*
@@ -87,28 +96,39 @@ This handler will be responsible for proxying out the GET requests that the serv
 */
 func (c *Controller) Get(ctx *gin.Context) {
 	incomingPath := ctx.Param("ProxiedPath")
-	for idx := range staticRoutes {
-		if strings.Contains(incomingPath, staticRoutes[idx]) {
+	routeSplit := strings.Split(incomingPath, "/")
+	if len(routeSplit) > 2 {
+		baseRoute := strings.Join(routeSplit[:len(routeSplit)-1], "/")
+
+		_, ok := c.RouteMaps[c.Config.AltAllowedDomain].RouteSet[baseRoute]
+		if ok {
 			data, ctype, rcode, err := c.RetrieveStaticResource(ctx.Request.Method, incomingPath, ctx)
 			if err != nil {
-				log.Fatal(err, "reroute to the static domain")
+				log.Fatal(err, " failed to reroute to the static domain")
 			}
 			ctx.Data(rcode, ctype, data)
 			return
 		}
 	}
-	for idx := range apiRoutes {
-		if strings.Contains(incomingPath, apiRoutes[idx]) {
-			data, ctype, err := c.SiteauditApiCall(ctx.Request.Method, incomingPath, ctx.Request.URL.RawQuery, ctx.Request.Body, ctx)
-			if err != nil {
-				log.Fatal(err, "failed to route to the root domain")
-			}
-			ctx.Data(200, ctype, data)
-			return
+	_, ok := c.RouteMaps[c.Config.AltAllowedDomain].RouteSet[incomingPath]
+	if ok {
+		data, ctype, rcode, err := c.RetrieveStaticResource(ctx.Request.Method, incomingPath, ctx)
+		if err != nil {
+			log.Fatal(err, " failed to reroute to the static domain")
 		}
+		ctx.Data(rcode, ctype, data)
+		return
 	}
-
-	data, ctype, rcode, err := c.SemrushGeneric(ctx.Request.Method, incomingPath, ctx.Request.Body, ctx)
+	_, ok = c.RouteMaps[c.Config.AllowedDomain].RouteSet[incomingPath]
+	if ok {
+		data, ctype, rcode, err := c.SiteauditApiCall(ctx.Request.Method, incomingPath, ctx.Request.URL.RawQuery, ctx.Request.Body, ctx)
+		if err != nil {
+			log.Fatal(err, " failed to route to the root domain")
+		}
+		ctx.Data(rcode, ctype, data)
+		return
+	}
+	data, ctype, rcode, err := c.SemrushGeneric(ctx)
 	if err != nil {
 		ctx.JSON(rcode, map[string]string{
 			"Error": err.Error(),

pkg/include.go

@@ -3,8 +3,11 @@ package httpserver
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
+	"path"
 )
 
 type HttpServerConfig struct {
@@ -18,6 +21,7 @@ type HttpServerConfig struct {
 	UserAgent            string `json:"user_agent"`
 	UseSsl               bool   `json:"use_ssl"`
 	ProxyAddr            string `json:"proxy_addr"`
+	RouteMapDir          string `json:"routemap_dir"`
 	FullProxyDomain      string // the domain name of the proxied site with the protocol
 	CookieJar            []*http.Cookie
 	PhpSession           *http.Cookie
@@ -34,6 +38,12 @@ type Cookie struct {
 	IncludeSub bool   `json:"include_sub"`
 }
 
+type RouteMapping struct {
+	DomainName string   `json:"domain_name"`
+	UriPaths   []string `json:"uri_paths"`
+	RouteSet   map[string]struct{}
+}
+
 /*
 Reads the server configuration file, along with the cookie file so that the correlated account can be
 accessed through the proxy
@@ -86,3 +96,38 @@ func ReadConfig(loc string) (*HttpServerConfig, error) {
 	return &cfg, err
 
 }
+
+func ReadRouteMap(loc string) *RouteMapping {
+	f, err := os.ReadFile(loc)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var mapfile RouteMapping
+	err = json.Unmarshal(f, &mapfile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rmap := map[string]struct{}{}
+	for idx := range mapfile.UriPaths {
+		rmap[mapfile.UriPaths[idx]] = struct{}{}
+	}
+	mapfile.RouteSet = rmap
+
+	return &mapfile
+
+}
+
+func PopulateRouteMaps(dir string) map[string]*RouteMapping {
+
+	routeMaps := map[string]*RouteMapping{}
+	fnames, err := ioutil.ReadDir(dir)
+	if err != nil {
+		log.Fatal("Failed to read the routemap directory: ", err)
+	}
+	for _, f := range fnames {
+		mapping := ReadRouteMap(path.Join(dir, f.Name()))
+		routeMaps[f.Name()] = mapping
+
+	}
+	return routeMaps
+}

pkg/routing.go

@@ -10,8 +10,8 @@ Registers the exposed route to utilize the handler function
 	:param e: pointer to a gin.Engine struct
 	:param cfg: pointer to an HttpServerConfig struct
 */
-func RegisterRoutes(e *gin.Engine, cfg *HttpServerConfig) {
-	c := NewController(cfg)
+func RegisterRoutes(e *gin.Engine, cfg *HttpServerConfig, rmaps map[string]*RouteMapping) {
+	c := NewController(cfg, rmaps)
 	web := e.Group("")
 	web.Any("/*ProxiedPath", c.Get)