working on bullshit with cookie handling...

.gitignore

@@ -38,4 +38,8 @@ inc/**
 # cookies.json file
 cookies.json
 
+# anything .pem
+*.pem
 
+# ignoring all temp files
+temp_*

cmd/server.go

@@ -16,6 +16,6 @@ func main() {
 	fmt.Printf("%+v\n", cfg)
 	e := gin.Default()
 	httpserver.RegisterRoutes(e, cfg)
-	e.Run(fmt.Sprintf("%s:%v", "0.0.0.0", cfg.HttpPort))
+	e.RunTLS(fmt.Sprintf("%s:%v", "0.0.0.0", cfg.HttpsPort), "./certificate.pem", "privatekey.pem")
 
 }

pkg/controller.go

@@ -5,10 +5,12 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/http/cookiejar"
 	"net/url"
 	"strings"
 
 	"github.com/gin-gonic/gin"
+	"golang.org/x/net/publicsuffix"
 )
 
 // Implementing a 'set'
@@ -22,55 +24,51 @@ var NonmutableHeaders = map[string]struct{}{
 }
 
 type Controller struct {
-	Config *HttpServerConfig
-	Client *http.Client
+	Config  *HttpServerConfig
+	Client  *http.Client
+	SiteUrl *url.URL
 }
 
 type ProxyCookies struct {
 	ck map[*url.URL][]*http.Cookie
 }
 
-func (p *ProxyCookies) SetCookies(u *url.URL, cookies []*http.Cookie) {
-	p.ck[u] = cookies
-}
-
-func (p *ProxyCookies) Cookies(u *url.URL) []*http.Cookie {
-	cookies, ok := p.ck[u]
-	if !ok {
-		return []*http.Cookie{
-			nil,
-		}
-	} else {
-		return cookies
-	}
-}
-
 /*
 Returns a new Controller struct to register routes to the gin router
 
 	:param cfg: A pointer to an HttpServerConfig struct
 */
 func NewController(cfg *HttpServerConfig) *Controller {
-	ckJar := &ProxyCookies{}
-	for idx := range cfg.CookieJar {
-		ck := cfg.CookieJar[idx]
-		url, err := url.Parse(ck.Domain)
-		if err != nil {
-			log.Fatal("required cookie could not be patsed: ", ck, err)
-		}
-		ckJar.SetCookies(url, ck)
+	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	sessCookies := cfg.CookieJar
+	domain, err := url.Parse("https://www.semrush.com")
+	if err != nil {
+		log.Fatal(err)
 	}
+	jar.SetCookies(domain, sessCookies)
 
-	return &Controller{Config: cfg, Client: &http.Client{Jar: ckJar}}
+	return &Controller{Config: cfg, Client: &http.Client{Jar: jar}, SiteUrl: domain}
 }
 
 /*
 This handler will be responsible for proxying out the GET requests that the server recieves
 */
 func (c *Controller) Get(ctx *gin.Context) {
-
-	url := fmt.Sprintf("https://%s%s", c.Config.AllowedDomain, ctx.Param("ProxiedPath"))
-	req, err := http.NewRequest(ctx.Request.Method, url, ctx.Request.Body)
+	var allowedDomain string
+	//	if ctx.Param("ProxiedPath") == "/siteaudit/i18n/messages_en.fd3cadbc.json" {
+	//		allowedDomain = "static.semrush.com"
+	//	}
+	//if strings.Contains(ctx.Param("ProxiedPath"), "/siteaudit/api") {
+	//		allowedDomain = "static.semrush.com"
+	//	} else {
+	allowedDomain = c.Config.AllowedDomain
+	//	}
+	reqUrl := fmt.Sprintf("https://%s%s", allowedDomain, ctx.Param("ProxiedPath"))
+	req, err := http.NewRequest(ctx.Request.Method, reqUrl, ctx.Request.Body)
 	if err != nil {
 		ctx.JSON(500, map[string]string{
 			"Error": "Error creating the request.",
@@ -79,13 +77,22 @@ func (c *Controller) Get(ctx *gin.Context) {
 		return
 	}
 	req.URL.Path = ctx.Param("ProxiedPath")
+	cookie, err := ctx.Cookie("csrftoken")
+	fmt.Printf("------------- INBOUND COOKIE --------\n%+v\n%s", cookie, err)
 	req.Header.Add("User-Agent", c.Config.UserAgent)
-	req.Header.Set("Referer", c.Config.AllowedDomain)
+	req.AddCookie(&http.Cookie{Name: "csrftoken", Value: cookie, Path: "/siteaudit", MaxAge: 3600, HttpOnly: true, Secure: true, SameSite: http.SameSiteNoneMode})
+	if strings.Split(ctx.Param("ProxiedPath"), "?")[0] == "/siteaudit/api/campaigns/seolist" {
+		req.Header.Set("Referer", fmt.Sprintf("https://%s/siteaudit/", c.Config.AllowedDomain))
+		req.Header.Set("referrer", fmt.Sprintf("https://%s/siteaudit/", c.Config.AllowedDomain))
+
+	} else {
+		req.Header.Set("Referer", c.Config.AllowedDomain)
+	}
 	if ctx.Param("ProxiedPath") == "/" {
-		req.Header.Add("Location", "https://sem.bunnytools.shop/analytics/overview/")
+		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/overview/")
 	}
 	if ctx.Param("ProxiedPath") == "/_compatibility/traffic/overview/" {
-		req.Header.Add("Location", "https://sem.bunnytools.shop/analytics/traffic/overview/ebay.com")
+		ctx.Header("Location", "https://sem.bunnytools.shop/analytics/traffic/overview/ebay.com")
 	}
 	for k, v := range ctx.Request.Header {
 		_, ok := NonmutableHeaders[k]
@@ -93,7 +100,6 @@ func (c *Controller) Get(ctx *gin.Context) {
 			req.Header.Add(k, v[0])
 		}
 	}
-	fmt.Printf("%+v\n", req.Header)
 	resp, err := c.Client.Do(req)
 	if err != nil {
 		ctx.JSON(500, map[string]string{
@@ -111,21 +117,21 @@ func (c *Controller) Get(ctx *gin.Context) {
 		})
 		return
 	}
-	responseCookies := resp.Cookies()
-	for idx := range responseCookies {
-		url, err := url.Parse(ck.Domain)
-		if err != nil {
-			fmt.Printf("required cookie could not be patsed: %+v\n, error: %s\n", ck, err)
+	for k, v := range resp.Header {
+		_, ok := NonmutableHeaders[k]
+		if !ok {
+			ctx.Header(k, v[0])
 		}
-
-		c.Client.Jar.SetCookies(url, responseCookies[idx])
 	}
 
+	fmt.Printf("----------------- HEADERS GOING OUT ---------\n%+v\n", req.Header)
+	fmt.Printf("-------------- RESPONSE HEADERS ------------\n%+v\n", resp.Header)
+	fmt.Printf("-------------- COOKIES GOING OUT ------------ \n%+v\n", req.Cookies())
 	newBody := strings.ReplaceAll(string(b), "\"srf-browser-unhappy\"", "\"srf-browser-unhappy\" style=\"display:none;\"")
 	newBody = strings.ReplaceAll(newBody, "\"srf-navbar__right\"", "\"srf-navbar__right\" style=\"display:none;\"")
 	newBody = strings.ReplaceAll(newBody, "<footer", "<footer style=\"display:none;\"")
 	newBody = strings.ReplaceAll(newBody, "\"srf-report-sidebar-extras\"", "\"srf-report-sidebar-extra\" style=\"display:none;\"")
 	newBody = strings.ReplaceAll(newBody, "www.semrush.com", "sem.bunnytools.shop")
+	//	newBody = strings.ReplaceAll(newBody, "static.semrush.com", "sem.bunnytools.shop")
 	ctx.Data(resp.StatusCode, resp.Header.Get("Content-Type"), []byte(newBody))
-
 }

pkg/include.go

@@ -2,6 +2,7 @@ package httpserver
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 )
@@ -65,6 +66,7 @@ func ReadConfig(loc string) (*HttpServerConfig, error) {
 
 	}
 
+	fmt.Printf("%+v\n", cfg)
 	return &cfg, err
 
 }

pkg/routing.go

@@ -1,6 +1,8 @@
 package httpserver
 
-import "github.com/gin-gonic/gin"
+import (
+	"github.com/gin-gonic/gin"
+)
 
 /*
 Registers the exposed route to utilize the handler function