still working on cleanup, making sure things are routing properly

cmd/server.go

@@ -25,6 +25,6 @@ func main() {
 	e.Use(cors.New(config))
 
 	httpserver.RegisterRoutes(e, cfg)
-	e.RunTLS(fmt.Sprintf("%s:%v", "0.0.0.0", cfg.HttpsPort), "./certificate.pem", "privatekey.pem")
+	e.RunTLS(fmt.Sprintf("%s:%v", "0.0.0.0", cfg.HttpsPort), "/etc/letsencrypt/live/void-society.online/fullchain.pem", "/etc/letsencrypt/live/void-society.online/privkey.pem")
 
 }

pkg/client.go

@@ -5,33 +5,33 @@ import (
 	"io"
 	"net/http"
 	"strings"
+
+	"github.com/gin-gonic/gin"
 )
 
 /*
-	 Retrieve the site audit config file from Semrush
-		returns a byte array of the body, the content type of the resp, and an error
+Retrieve the site audit config file from Semrush
+
+	returns a byte array of the body, the content type of the resp, and an error
 */
-func (c *Controller) RetrieveStaticResource(path string) ([]byte, string, error) {
+func (c *Controller) RetrieveStaticResource(path string, ctx *gin.Context) ([]byte, string, int, error) {
 	url := fmt.Sprintf("https://static.semrush.com%s", path)
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		return nil, "", err
+		return nil, "", 500, err
 	}
-	req.AddCookie(c.Config.PhpSession)
-	req.AddCookie(c.Config.SsoToken)
-	req.Header.Set("User-Agent", c.Config.UserAgent)
-	req.Header.Set("Referer", "https://www.semrush.com")
-	req.Header.Set("Origin", "https://www.semrush.com")
+	c.setHeaders(req, ctx)
+
 	resp, err := c.Client.Do(req)
 	if err != nil {
-		return nil, "", err
+		return nil, "", 500, err
 	}
 	defer resp.Body.Close()
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, "", err
+		return nil, "", 500, err
 	}
-	return b, resp.Header.Get("content-type"), nil
+	return b, resp.Header.Get("content-type"), resp.StatusCode, nil
 
 }
 
@@ -40,20 +40,17 @@ Perform a call against the siteaudit api
 
 	:param path: the URI path with the query
 	:param query: the query to add to the request
+	:param body: an io.Reader to push into the request body
 	:returns a byte array of the response, the content type and an error
 */
-func (c *Controller) SiteauditApiCall(method string, path string, query string, body io.Reader) ([]byte, string, error) {
-	query = strings.ReplaceAll(query, "https://sem.bunnytools.shop", "https://www.semrush.com")
-	url := fmt.Sprintf("https://www.semrush.com%s?%s", path, query)
+func (c *Controller) SiteauditApiCall(method string, path string, query string, body io.Reader, ctx *gin.Context) ([]byte, string, error) {
+	query = strings.ReplaceAll(query, c.Config.FullProxyDomain, c.Config.FullDomain)
+	url := fmt.Sprintf("%s%s?%s", c.Config.FullDomain, path, query)
 	req, err := http.NewRequest(method, url, body)
 	if err != nil {
 		return nil, "", err
 	}
-	req.AddCookie(c.Config.PhpSession)
-	req.AddCookie(c.Config.SsoToken)
-	req.Header.Set("User-Agent", c.Config.UserAgent)
-	req.Header.Set("Referer", "https://www.semrush.com")
-	req.Header.Set("Origin", "https://www.semrush.com")
+	c.setHeaders(req, ctx)
 	resp, err := c.Client.Do(req)
 	if err != nil {
 		return nil, "", err
@@ -70,6 +67,53 @@ func (c *Controller) SiteauditApiCall(method string, path string, query string,
 /*
 Generic site call to the semrush site
 */
-func (c *Controller) SemrushGeneric() ([]byte, string, error) {
-	// TODO: make this working lol
+func (c *Controller) SemrushGeneric(method string, path string, body io.Reader, ctx *gin.Context) ([]byte, string, int, error) {
+
+	url := fmt.Sprintf("%s%s", c.Config.FullDomain, path)
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, "", 500, err
+	}
+	c.setHeaders(req, ctx)
+	resp, err := c.Client.Do(req)
+	if err != nil {
+		return nil, "", 500, err
+	}
+	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, "", 500, err
+	}
+
+	for k, v := range resp.Header {
+		_, ok := NonmutableHeaders[k]
+		if !ok {
+			ctx.Header(k, v[0])
+		}
+	}
+
+	return b, resp.Header.Get("content-type"), resp.StatusCode, nil
+
+}
+
+/*
+Sets the request headers to whatever is defined in this private method
+
+	:param req: a pointer to an HTTP request
+*/
+func (c *Controller) setHeaders(req *http.Request, ctx *gin.Context) {
+
+	req.AddCookie(c.Config.PhpSession)
+	req.AddCookie(c.Config.SsoToken)
+	req.Header.Set("User-Agent", c.Config.UserAgent)
+	req.Header.Set("Referer", c.Config.FullDomain)
+	req.Header.Set("Origin", c.Config.FullDomain)
+
+	for k, v := range ctx.Request.Header {
+		_, ok := NonmutableHeaders[k]
+		if !ok {
+			req.Header.Add(k, v[0])
+		}
+	}
+
 }

pkg/controller.go

@@ -1,8 +1,6 @@
 package httpserver
 
 import (
-	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"net/http/cookiejar"
@@ -24,6 +22,7 @@ var staticRoutes = [...]string{
 	"/keyword-overview/",
 	"/keyword-gap/",
 	"/oti/prod/organic_traffic_insights/",
+	"/oti/prod/organic-traffic-insights",
 	"/ajst/",
 	"/listing-management/landings/",
 	"/listing-management/landing-reviews/",
@@ -73,7 +72,7 @@ func NewController(cfg *HttpServerConfig) *Controller {
 	}
 
 	sessCookies := cfg.CookieJar
-	domain, err := url.Parse("https://www.semrush.com")
+	domain, err := url.Parse(cfg.FullDomain)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -89,78 +88,44 @@ func (c *Controller) Get(ctx *gin.Context) {
 	incomingPath := ctx.Param("ProxiedPath")
 	for idx := range staticRoutes {
 		if strings.Contains(incomingPath, staticRoutes[idx]) {
-			data, ctype, err := c.RetrieveStaticResource(incomingPath)
+			data, ctype, rcode, err := c.RetrieveStaticResource(incomingPath, ctx)
 			if err != nil {
-				log.Fatal(err, "failed to get site config")
+				log.Fatal(err, "reroute to the static domain")
 			}
-			ctx.Data(200, ctype, data)
+			ctx.Data(rcode, ctype, data)
 			return
 		}
 	}
 	for idx := range apiRoutes {
 		if strings.Contains(incomingPath, apiRoutes[idx]) {
-			data, ctype, err := c.SiteauditApiCall(ctx.Request.Method, incomingPath, ctx.Request.URL.RawQuery, ctx.Request.Body)
+			data, ctype, err := c.SiteauditApiCall(ctx.Request.Method, incomingPath, ctx.Request.URL.RawQuery, ctx.Request.Body, ctx)
 			if err != nil {
-				log.Fatal(err, "failed to get site config")
+				log.Fatal(err, "failed to route to the root domain")
 			}
 			ctx.Data(200, ctype, data)
 			return
 		}
 	}
 
-	reqUrl := fmt.Sprintf("https://%s%s", c.Config.AllowedDomain, incomingPath)
-	req, err := http.NewRequest(ctx.Request.Method, reqUrl, ctx.Request.Body)
+	data, ctype, rcode, err := c.SemrushGeneric(ctx.Request.Method, incomingPath, ctx.Request.Body, ctx)
 	if err != nil {
-		ctx.JSON(500, map[string]string{
-			"Error": "Error creating the request.",
-			"Msg:":  err.Error(),
+		ctx.JSON(rcode, map[string]string{
+			"Error": err.Error(),
 		})
 		return
 	}
-	req.URL.Path = incomingPath
-	req.Header.Add("User-Agent", c.Config.UserAgent)
-	req.Header.Set("Referer", c.Config.AllowedDomain)
+
 	if incomingPath == "/" {
 		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/overview/")
 	}
 	if incomingPath == "/_compatibility/traffic/overview/" {
 		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/traffic/overview/ebay.com")
 	}
-	for k, v := range ctx.Request.Header {
-		_, ok := NonmutableHeaders[k]
-		if !ok {
-			req.Header.Add(k, v[0])
-		}
-	}
-	resp, err := c.Client.Do(req)
-	if err != nil {
-		ctx.JSON(500, map[string]string{
-			"Error": "Error creating the request.",
-			"Msg:":  err.Error(),
-		})
-		return
-	}
-	defer resp.Body.Close()
-	b, err := io.ReadAll(resp.Body)
-	if err != nil {
-		ctx.JSON(500, map[string]string{
-			"Error": "Error creating the request.",
-			"Msg:":  err.Error(),
-		})
-		return
-	}
-	for k, v := range resp.Header {
-		_, ok := NonmutableHeaders[k]
-		if !ok {
-			ctx.Header(k, v[0])
-		}
-	}
-
-	newBody := strings.ReplaceAll(string(b), "\"srf-browser-unhappy\"", "\"srf-browser-unhappy\" style=\"display:none;\"")
+	newBody := strings.ReplaceAll(string(data), "\"srf-browser-unhappy\"", "\"srf-browser-unhappy\" style=\"display:none;\"")
 	newBody = strings.ReplaceAll(newBody, "\"srf-navbar__right\"", "\"srf-navbar__right\" style=\"display:none;\"")
 	newBody = strings.ReplaceAll(newBody, "<footer", "<footer style=\"display:none;\"")
 	newBody = strings.ReplaceAll(newBody, "\"srf-report-sidebar-extras\"", "\"srf-report-sidebar-extra\" style=\"display:none;\"")
-	newBody = strings.ReplaceAll(newBody, "www.semrush.com", "sem.bunnytools.shop")
-	newBody = strings.ReplaceAll(newBody, "static.semrush.com", "sem.bunnytools.shop")
-	ctx.Data(resp.StatusCode, resp.Header.Get("Content-Type"), []byte(newBody))
+	newBody = strings.ReplaceAll(newBody, c.Config.AllowedDomain, c.Config.ProxyAddr)
+	newBody = strings.ReplaceAll(newBody, "static.semrush.com", c.Config.ProxyAddr)
+	ctx.Data(rcode, ctype, []byte(newBody))
 }

pkg/include.go

@@ -8,15 +8,18 @@ import (
 )
 
 type HttpServerConfig struct {
-	HttpPort      int    `json:"http_port"`
-	HttpsPort     int    `json:"https_port"`
-	AllowedDomain string `json:"allowed_domain"`
-	UserAgent     string `json:"user_agent"`
-	UseSsl        bool   `json:"use_ssl"`
-	ProxyAddr     string `json:"proxy_addr"`
-	CookieJar     []*http.Cookie
-	PhpSession    *http.Cookie
-	SsoToken      *http.Cookie
+	HttpPort        int    `json:"http_port"`
+	HttpsPort       int    `json:"https_port"`
+	AllowedDomain   string `json:"allowed_domain"`
+	FullDomain      string // The domain name with the protocol before it
+	Proto           string `json:"proto"` // http/https
+	UserAgent       string `json:"user_agent"`
+	UseSsl          bool   `json:"use_ssl"`
+	ProxyAddr       string `json:"proxy_addr"`
+	FullProxyDomain string // the domain name of the proxied site with the protocol
+	CookieJar       []*http.Cookie
+	PhpSession      *http.Cookie
+	SsoToken        *http.Cookie
 }
 
 type Cookie struct {
@@ -50,6 +53,8 @@ func ReadConfig(loc string) (*HttpServerConfig, error) {
 	if err != nil {
 		return nil, err
 	}
+	cfg.FullDomain = fmt.Sprintf("%s://%s", cfg.Proto, cfg.AllowedDomain)
+	cfg.FullProxyDomain = fmt.Sprintf("%s://%s", cfg.Proto, cfg.ProxyAddr)
 	var cookies []Cookie
 	err = json.Unmarshal(cf, &cookies)
 	if err != nil {