working on refactoring this into an API discovery tool.

cmd/http-wokou/http-wokou.go

@@ -27,13 +27,13 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
 	"os"
 
 	httpserver "git.aetherial.dev/aeth/http-proxy/pkg"
-	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
 )
 
@@ -45,11 +45,11 @@ var licenseMsgLong = "\n		GNU GENERAL PUBLIC LICENSE\n		Version 3, 29 June 2007\
 
 func main() {
 	caching := flag.Bool("cache", false, "Supply this argument to turn on caching")
+	makeConfig := flag.Bool("mk-config", false, "Pass this flag to generate a blank config file")
 	licenseInfo := flag.Bool("license", false, "Pass this flag to display license and warantee information.")
 	redistInfo := flag.Bool("redist", false, "Pass this flag to display redistribution information.")
 	configFile := flag.String("config", ".config", "supply the path to a configuration file for the proxy to use")
-	tokenCode := flag.String("tokencode", "1234", "Supply this flag followed with a password/code to use for authenticating to the cookie update page.")
-	tokenSaveLoc := flag.String("token-loc", "./token", "Pass a path to this flag to specify a token save location")
+	//	useSsl := flag.Bool("ssl", false, "pass this to use the ssl files pointed to in the config")
 	flag.Parse()
 
 	if *licenseInfo {
@@ -61,21 +61,28 @@ func main() {
 		os.Exit(0)
 	}
 	fmt.Println(licenseMsg)
+
+	if *makeConfig {
+		out := httpserver.HttpServerConfig{}
+		b, err := json.Marshal(&out)
+		if err != nil {
+			log.Fatal(err)
+		}
+		os.WriteFile(".config", b, os.ModePerm)
+		rm := httpserver.RouteMap{}
+		b, _ = json.Marshal(rm)
+		os.WriteFile("./routemap.json", b, os.ModePerm)
+	}
+
 	cfg, err := httpserver.ReadConfig(*configFile)
 	if err != nil {
 		log.Fatal("Couldnt read config: ", err)
 	}
 	cfg.Caching = *caching
-	cfg.TkUpdateCode = *tokenCode
-	cfg.TokenSaveLoc = *tokenSaveLoc
 	e := gin.Default()
 	e.SetTrustedProxies(nil)
-	config := cors.DefaultConfig()
-	config.AllowOrigins = cfg.CorsHosts
-	e.Use(cors.New(config))
-	rmaps := httpserver.ReadRouteMap(cfg.RouteMapPath)
-	httpserver.RegisterRoutes(e, cfg, rmaps)
-
-	e.RunTLS(fmt.Sprintf("%s:%v", cfg.ListeningIp, cfg.HttpsPort), cfg.SslPemFile, cfg.SslKeyFile)
-
+	//config := cors.DefaultConfig()
+	//e.Use(cors.New(config))
+	httpserver.RegisterRoutes(e, cfg)
+	e.Run(fmt.Sprintf("%s:%v", cfg.ListeningIp, cfg.HttpPort))
 }

pkg/client.go

@@ -35,63 +35,42 @@ import (
 	"sync"
 )
 
+type ResponseCarrier struct {
+	Data       []byte
+	Headers    http.Header
+	StatusCode int
+	Error      error
+}
+
 /*
 Generic site call to an upstream server
 */
-func (c *Controller) RequestGeneric(method string, host string, path string, hdrs *http.Header, body io.Reader) ([]byte, *http.Header, int, error) {
-	reqUrl := fmt.Sprintf("https://%s%s", host, path)
-	if method == "POST" {
-		req, err := http.NewRequest(method, reqUrl, c.requestBodyRewrites(body))
-		if err != nil {
-			return nil, nil, 500, err
-		}
-		c.setHeaders(req, hdrs)
-		resp, err := c.Client.Do(req)
-		if err != nil {
-			return nil, nil, 500, err
-		}
-		defer resp.Body.Close()
-		b, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return nil, nil, 500, err
-		}
-
-		return c.pageMod(b), &resp.Header, resp.StatusCode, nil
-
-	}
+func (c *Controller) RequestGeneric(method string, host string, path string, hdrs *http.Header, body io.Reader) ResponseCarrier {
+	reqUrl := fmt.Sprintf("http://%s%s", host, path)
 	req, err := http.NewRequest(method, reqUrl, body)
 	if err != nil {
-		return nil, nil, 500, err
+		return ResponseCarrier{Data: nil, Headers: nil, StatusCode: 500, Error: err}
 	}
-	c.setHeaders(req, hdrs)
 	resp, err := c.Client.Do(req)
 	if err != nil {
-		return nil, nil, 500, err
+		return ResponseCarrier{Data: nil, Headers: nil, StatusCode: 500, Error: err}
 	}
 	defer resp.Body.Close()
 	b, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, nil, 500, err
+		return ResponseCarrier{Data: nil, Headers: resp.Header, StatusCode: resp.StatusCode, Error: err}
 	}
-	var data []byte
-
-	_, ok := c.PageMods.Bypass[path]
-	if ok {
-		data = b
-	} else {
-		data = c.pageMod(b)
 
-	}
 	if c.Config.Caching {
 		if !strings.Contains(path, "?") {
 			if resp.StatusCode == 200 {
 				if method == "GET" {
-					c.CacheResource(path, NewCachedResource(data, &resp.Header, resp.StatusCode))
+					c.CacheResource(path, NewCachedResource(b, &resp.Header, resp.StatusCode))
 				}
 			}
 		}
 	}
-	return data, &resp.Header, resp.StatusCode, nil
+	return ResponseCarrier{Data: b, Headers: resp.Header, StatusCode: resp.StatusCode, Error: nil}
 
 }
 
@@ -101,11 +80,11 @@ func (c *Controller) TryHosts(method string, path string, hdrs *http.Header, bod
 		wg.Add(1)
 		go func(method string, host string, path string, hdrs *http.Header, body io.Reader) {
 			defer wg.Done()
-			_, _, rcode, err := c.RequestGeneric(method, host, path, hdrs, body)
-			if err != nil {
-				log.Fatal("Fatal Error creating request in a RequestGeneric method: ", err)
+			resp := c.RequestGeneric(method, host, path, hdrs, body)
+			if resp.Error != nil {
+				log.Fatal("Fatal Error creating request in a RequestGeneric method: ", resp.Error)
 			}
-			if rcode == 200 {
+			if resp.StatusCode == 200 {
 				basePath := strings.Split(path, "?")[0]
 				c.RouteMaps.MapUriToDomain(basePath, host)
 			}
@@ -122,11 +101,10 @@ Sets the request headers to whatever is defined in this private method
 */
 func (c *Controller) setHeaders(req *http.Request, hdrs *http.Header) {
 
-	req.AddCookie(c.Config.PhpSession)
-	req.AddCookie(c.Config.SsoToken)
-	req.Header.Set("User-Agent", c.Config.UserAgent)
-	req.Header.Set("Referer", c.Config.FullDomain)
-	req.Header.Set("Origin", c.Config.FullDomain)
+	//req.AddCookie(c.Config.PhpSession)
+	//req.Header.Set("User-Agent", c.Config.UserAgent)
+	//req.Header.Set("Referer", c.Config.FullDomain)
+	//req.Header.Set("Origin", c.Config.FullDomain)
 	for k, v := range *hdrs {
 		_, ok := NonmutableHeaders[k]
 		if !ok {

pkg/configuration.go

@@ -28,7 +28,6 @@ package httpserver
 
 import (
 	"encoding/json"
-	"fmt"
 	"log"
 	"net/http"
 	"os"
@@ -38,49 +37,19 @@ import (
 )
 
 type HttpServerConfig struct {
-	HttpPort             int             `json:"http_port"`
-	HttpsPort            int             `json:"https_port"`
-	AllowedDomain        string          `json:"allowed_domain"`
-	FullDomain           string          // The domain name with the protocol before it
-	AltAllowedDomain     string          `json:"alt_allowed_domain"` // alternate domain that resources are sourced from
-	FullAltAllowedDomain string          // the alt domain with the protocol
-	Proto                string          `json:"proto"` // http/https
-	UserAgent            string          `json:"user_agent"`
-	SslPemFile           string          `json:"ssl_pem_file"`
-	SslKeyFile           string          `json:"ssl_key_file"`
-	ListeningIp          string          `json:"listening_ip"`
-	ProxyAddr            string          `json:"proxy_addr"`
-	RouteMapPath         string          `json:"route_map_path"`
-	PageModPath          string          `json:"page_mod_path"`
-	CustomFServePath     string          `json:"custom_fileserve_cfg_path"`
-	CookieFile           string          `json:"cookie_file"`
-	FullProxyDomain      string          // the domain name of the proxied site with the protocol
-	KnownHosts           []string        `json:"known_hosts"`
-	CorsHosts            []string        `json:"cors_hosts"`
-	Redirects            []*RedirectRule `json:"redirects"`
-	Caching              bool
-	TkUpdateCode         string
-	CustomFserve         *CustomFileServer
-	TokenSaveLoc         string
-	CookieJar            []*http.Cookie
-	PhpSession           *http.Cookie
-	SsoToken             *http.Cookie
-}
-
-type CustomFile struct {
-	Request     string `json:"request"`
-	Serve       string `json:"serve"`
-	ContentType string `json:"content-type"`
-	FileData    []byte
-}
-
-type CustomFileServer struct {
-	Config []*CustomFile `json:"config"`
-}
-
-type RedirectRule struct {
-	From string `json:"from"`
-	To   string `json:"to"`
+	HttpPort     int      `json:"http_port"`
+	HttpsPort    int      `json:"https_port"`
+	ServerDomain string   // This is the domain that will be used by the client
+	ProxyDomain  string   // This will be the domain that is being proxied, i.e. the downstream domain
+	SslPemFile   string   `json:"ssl_pem_file"`
+	SslKeyFile   string   `json:"ssl_key_file"`
+	ListeningIp  string   `json:"listening_ip"`
+	KnownHosts   []string `json:"known_hosts"`
+	RouteMapPath string   `json:"route_map_path"` // path to the routemaps
+	Caching      bool
+	CookieFile   string `json:"cookie_file"` // path to a cookie for the outbound client
+	CookieJar    []*http.Cookie
+	Home         string `json:"home"` // the location on the filesystem to save the output config to
 }
 
 type Cookie struct {
@@ -99,42 +68,29 @@ accessed through the proxy
 
 	:param loc: the location of the config file
 */
-func ReadConfig(loc string) (*HttpServerConfig, error) {
+func ReadConfig(loc string) (HttpServerConfig, error) {
 
 	f, err := os.ReadFile(loc)
 	if err != nil {
-		return nil, err
+		return HttpServerConfig{}, err
 	}
 	var cfg HttpServerConfig
 	err = json.Unmarshal(f, &cfg)
 	if err != nil {
-		return nil, err
+		return HttpServerConfig{}, err
 	}
 	cf, err := os.ReadFile(cfg.CookieFile)
 	if err != nil {
-		return nil, err
+		return HttpServerConfig{}, err
 	}
-	cfg.CustomFserve = ReadCustomFiles(cfg.CustomFServePath)
-	cfg.FullDomain = fmt.Sprintf("%s://%s", cfg.Proto, cfg.AllowedDomain)
-	cfg.FullProxyDomain = fmt.Sprintf("%s://%s", cfg.Proto, cfg.ProxyAddr)
-	cfg.FullAltAllowedDomain = fmt.Sprintf("%s://%s", cfg.Proto, cfg.AltAllowedDomain)
 	var cookies []*http.Cookie
 	err = json.Unmarshal(cf, &cookies)
 	if err != nil {
-		return nil, err
-	}
-	for idx := range cookies {
-		if cookies[idx].Name == "PHPSESSID" {
-			cfg.PhpSession = cookies[idx]
-		}
-		if cookies[idx].Name == "sso_token" {
-			cfg.SsoToken = cookies[idx]
-		}
-
+		return HttpServerConfig{}, err
 	}
 	cfg.CookieJar = cookies
 
-	return &cfg, err
+	return cfg, err
 
 }
 
@@ -153,33 +109,3 @@ func ReadRouteMap(loc string) *RouteMap {
 	return &mapfile
 
 }
-
-/*
-Read in the custom file server configuration
-
-	:param loc: path to the custom file server config
-*/
-func ReadCustomFiles(loc string) *CustomFileServer {
-	b, err := os.ReadFile(loc)
-	if err != nil {
-		fmt.Printf("couldnt read custom config file: %s\n", err.Error())
-		return nil
-	}
-	var fserveCfg CustomFileServer
-	err = json.Unmarshal(b, &fserveCfg)
-	if err != nil {
-		log.Fatal("error loading in the config file: ", err)
-	}
-	for idx := range fserveCfg.Config {
-		_, err = os.Stat(fserveCfg.Config[idx].Serve)
-		if err != nil {
-			log.Fatal("Couldnt verify the existence of file: ", fserveCfg.Config[idx].Serve, " error: ", err)
-		}
-		b, err = os.ReadFile(fserveCfg.Config[idx].Serve)
-		if err != nil {
-			log.Fatal("Error reading in custom fileserver file: ", err)
-		}
-		fserveCfg.Config[idx].FileData = b
-	}
-	return &fserveCfg
-}

pkg/controller.go

@@ -32,7 +32,6 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
-	"os"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -56,11 +55,9 @@ type TokenUpdate struct {
 }
 
 type Controller struct {
-	Config    *HttpServerConfig
+	Config    HttpServerConfig
 	RouteMaps *RouteMap
-	PageMods  *AllPageMods
 	Client    *http.Client
-	SiteUrl   *url.URL
 	cache     *cache.Cache
 }
 
@@ -73,7 +70,7 @@ Returns a new Controller struct to register routes to the gin router
 
 	:param cfg: A pointer to an HttpServerConfig struct
 */
-func NewController(cfg *HttpServerConfig, routeMap *RouteMap) *Controller {
+func NewController(cfg HttpServerConfig, routeMap *RouteMap) *Controller {
 
 	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
 	if err != nil {
@@ -81,11 +78,10 @@ func NewController(cfg *HttpServerConfig, routeMap *RouteMap) *Controller {
 	}
 
 	sessCookies := cfg.CookieJar
-	domain, err := url.Parse(cfg.FullDomain)
+	domain, err := url.Parse(cfg.ProxyDomain)
 	if err != nil {
 		log.Fatal(err)
 	}
-	pgMod := LoadPageMods(cfg.PageModPath)
 	jar.SetCookies(domain, sessCookies)
 
 	var resCache *cache.Cache
@@ -96,8 +92,7 @@ func NewController(cfg *HttpServerConfig, routeMap *RouteMap) *Controller {
 		fmt.Printf("Starting server with resource caching DISABLED.\n")
 		resCache = nil
 	}
-	return &Controller{Config: cfg, Client: &http.Client{Jar: jar, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }},
-		SiteUrl: domain, cache: resCache, RouteMaps: routeMap, PageMods: pgMod}
+	return &Controller{Config: cfg, Client: &http.Client{Jar: jar, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }}, cache: resCache, RouteMaps: routeMap}
 }
 
 /*
@@ -105,28 +100,6 @@ This handler will be responsible for proxying out the GET requests that the serv
 */
 func (c *Controller) HandleAny(ctx *gin.Context) {
 	incomingPath := ctx.Param("ProxiedPath")
-	for idx := range c.Config.Redirects {
-		if incomingPath == c.Config.Redirects[idx].From {
-			ctx.Header("Location", c.Config.Redirects[idx].To)
-			ctx.Status(302)
-			return
-		}
-	}
-	if incomingPath == "/update" {
-		if ctx.Request.Method == "POST" {
-			c.UpdatePost(ctx)
-			return
-		}
-	}
-	if c.Config.CustomFserve != nil {
-		for idx := range c.Config.CustomFserve.Config {
-			if incomingPath == c.Config.CustomFserve.Config[idx].Request {
-				fmt.Print("Custom file server path hit.\n")
-				ctx.Data(200, c.Config.CustomFserve.Config[idx].ContentType, c.Config.CustomFserve.Config[idx].FileData)
-				return
-			}
-		}
-	}
 	if c.Config.Caching {
 		cacheHit := c.GetResource(incomingPath)
 		if cacheHit != nil {
@@ -145,50 +118,22 @@ func (c *Controller) HandleAny(ctx *gin.Context) {
 		}
 	}
 
-	dname, ok := c.RouteMaps.GetMappedDomain(incomingPath)
-	if ok { // below, RequestURI() returns the whole URI with the query
-		data, headers, rcode, err := c.RequestGeneric(ctx.Request.Method, dname, ctx.Request.URL.RequestURI(), &ctx.Request.Header, ctx.Request.Body)
-		if err != nil {
-			log.Fatal(err, " failed to route the request: ", incomingPath, " to the target domain: ", dname, " Error: ", err)
-		}
-		for k, v := range *headers {
-			_, ok := NonmutableHeaders[k]
-			if !ok {
-				ctx.Header(k, v[0])
-			}
-		}
-		ctx.Header("access-control-allow-origin", c.Config.FullProxyDomain)
-		ctx.Data(rcode, headers.Get("content-type"), data)
-		return
+	//dname, ok := c.RouteMaps.GetMappedDomain(incomingPath)
+	//if ok { // below, RequestURI() returns the whole URI with the query
+	resp := c.RequestGeneric(ctx.Request.Method, c.Config.ProxyDomain, ctx.Request.URL.RequestURI(), &ctx.Request.Header, ctx.Request.Body)
+	if resp.Error != nil {
+		log.Fatal(resp.Error, " failed to route the request: ", incomingPath, " to the target domain: ", ctx.Request.Host, " Error: ", resp.Error)
 	}
-
-	c.TryHosts(ctx.Request.Method, ctx.Request.URL.RequestURI(), &ctx.Request.Header, ctx.Request.Body, c.Config.KnownHosts)
-}
-
-/*
-This function handles the updating of cookie values, meant to be extendable down the road //  TODO: Make this more configurable
-
-	:param ctx: pointer to a gin Context struct
-*/
-func (c *Controller) UpdatePost(ctx *gin.Context) {
-	tk := TokenUpdate{
-		Code:    ctx.PostForm("code"),
-		Content: ctx.PostForm("content"),
-	}
-
-	if tk.Code != c.Config.TkUpdateCode {
-		ctx.JSON(401, map[string]string{
-			"msg": "UNAUTHORIZED",
-		})
-		return
-	}
-	err := os.WriteFile(c.Config.TokenSaveLoc, []byte(tk.Content), os.ModePerm)
-	if err != nil {
-		ctx.JSON(500, map[string]string{
-			"Error": fmt.Sprintf("couldnt write token to disk. Error: %s", err),
-		})
-		return
+	for k, v := range resp.Headers {
+		_, ok := NonmutableHeaders[k]
+		if !ok {
+			ctx.Header(k, v[0])
+		}
 	}
-	ctx.String(200, "Token updated.")
+	ctx.Header("access-control-allow-origin", c.Config.ServerDomain)
+	ctx.Data(resp.StatusCode, resp.Headers.Get("content-type"), resp.Data)
+	return
+	//}
 
+	//c.TryHosts(ctx.Request.Method, ctx.Request.URL.RequestURI(), &ctx.Request.Header, ctx.Request.Body, c.Config.KnownHosts)
 }

pkg/pagemod.go

@@ -30,7 +30,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"log"
 	"os"
 )
@@ -114,29 +113,7 @@ func LoadPageMods(loc string) *AllPageMods {
 /*
 Rewrite all occurences of these values into the response body
 */
-func (c *Controller) pageMod(data []byte) []byte {
-	for idx := range c.PageMods.Content {
-		if c.PageMods.Content[idx].Target == "content" {
-			data = bytes.ReplaceAll(data, []byte(c.PageMods.Content[idx].Search), []byte(c.PageMods.Content[idx].Sub))
-		}
-	}
+func pageMod(data []byte, server string, proxy string) []byte {
+	data = bytes.ReplaceAll(data, []byte(server), []byte(proxy))
 	return data
 }
-
-/*
-perform any request body rewrites as per described in the pagemod config
-
-	:param data: a byte array to modify
-*/
-func (c *Controller) requestBodyRewrites(data io.Reader) io.Reader {
-	b, err := io.ReadAll(data)
-	if err != nil {
-		log.Fatal("couldnt read POST body data: ", err)
-	}
-	for idx := range c.PageMods.Content {
-		if c.PageMods.Content[idx].Target == "body" {
-			b = bytes.ReplaceAll(b, []byte(c.PageMods.Content[idx].Search), []byte(c.PageMods.Content[idx].Sub))
-		}
-	}
-	return bytes.NewReader(b)
-}

pkg/routing.go

@@ -40,13 +40,13 @@ Registers the exposed route to utilize the handler function
 	:param e: pointer to a gin.Engine struct
 	:param cfg: pointer to an HttpServerConfig struct
 */
-func RegisterRoutes(e *gin.Engine, cfg *HttpServerConfig, rmaps *RouteMap) {
-	c := NewController(cfg, rmaps)
+func RegisterRoutes(e *gin.Engine, cfg HttpServerConfig) {
+	c := NewController(cfg, ReadRouteMap(cfg.RouteMapPath))
 	sigChanel := make(chan os.Signal)
 	signal.Notify(sigChanel, os.Interrupt, syscall.SIGINT)
 	go func(core *Controller) {
 		<-sigChanel
-		c.RouteMaps.ExportRouteMap(c.Config.RouteMapPath)
+		c.RouteMaps.ExportRouteMap(c.Config.Home)
 		os.Exit(1)
 	}(c)
 	web := e.Group("")