added htmx stuff and am trying to fine tune concurrency and memory usage problems

cmd/kyoketsu-web/kyoketsu-web.go

@@ -30,10 +30,9 @@ package main
 import (
 	"database/sql"
 	"flag"
-	"fmt"
 	"log"
+	"net/http"
 	"os"
-	"strings"
 	"sync"
 
 	kyoketsu "git.aetherial.dev/aeth/kyoketsu/pkg"
@@ -44,43 +43,29 @@ const dbfile = "sqlite.db"
 func main() {
 
 	port := flag.Int("port", 8080, "Select the port to run the server on")
-	addr := flag.String("ip", "", "Address to perform the initial network sweep on")
-
+	debug := flag.Bool("debug", false, "Pass this to start the pprof server")
 	flag.Parse()
 
-	if len(strings.Split(*addr, "/")) < 2 {
-		log.Fatal("You must pass an address that contains valid CIDR notation, i.e. '192.168.50.1/24'")
-	}
-
 	os.Remove(dbfile) // TODO: remove this once i add more smart interaction with the DB
 	db, err := sql.Open("sqlite3", dbfile)
 	if err != nil {
 		log.Fatal(err)
 	}
-
+	if *debug {
+		go func() {
+			log.Println(http.ListenAndServe("localhost:6060", nil))
+		}()
+	}
 	hostsRepo := kyoketsu.NewSQLiteRepo(db)
 	var wg sync.WaitGroup
 	wg.Add(1)
-	go kyoketsu.RunHttpServer(*port, hostsRepo)
+	go kyoketsu.RunHttpServer(*port, hostsRepo, kyoketsu.RetrieveScanDirectives())
 
 	if err = hostsRepo.Migrate(); err != nil {
 		log.Fatal(err)
 	}
 	log.Println("SUCCESS ::: SQLite database initiated, and open for writing.")
 
-	hosts, err := kyoketsu.NetSweep(*addr, kyoketsu.RetrieveScanDirectives().Pairs)
-	if err != nil {
-		log.Fatal(err)
-	}
-	for i := range hosts {
-		_, err := hostsRepo.Create(*hosts[i])
-		if err != nil {
-			log.Printf("Couldnt create new entry :( error: %s\n", err)
-
-		}
-		fmt.Println("SUCCESS ::: Host found. Adding to the database.")
-
-	}
 	wg.Wait()
 
 }

cmd/kyoketsu/kyoketsu.go

@@ -86,12 +86,12 @@ func main() {
 		wg.Add(1)
 		go func(target string, wg *sync.WaitGroup) {
 			defer wg.Done()
-			out := kyoketsu.PortWalk(target, kyoketsu.RetrieveScanDirectives().Pairs)
+			out := kyoketsu.PortWalk(target, kyoketsu.RetrieveScanDirectives())
 			if len(out.ListeningPorts) > 0 {
 				dns, _ := net.LookupAddr(out.IpAddress)
 				out.Fqdn = strings.Join(dns, ", ")
 
-				fmt.Println(" |-|-|-| :::: HOST FOUND :::: |-|-|-|\n==================||==================\n")
+				fmt.Print(" |-|-|-| :::: HOST FOUND :::: |-|-|-|\n==================||==================\n")
 				fmt.Printf("Hostname: %s\nIPv4 Address: %s\nPing Response?: %v\nListening Ports: %s\n=====================================\n", out.Fqdn, out.IpAddress, out.PingResponse, out.PortString)
 
 			}

go.mod

@@ -3,11 +3,13 @@ module git.aetherial.dev/aeth/kyoketsu
 go 1.21.1
 
 require (
-	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
+	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/go-ping/ping v1.1.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20240416155748-26353dc0451f // indirect
 	github.com/google/uuid v1.2.0 // indirect
+	github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
 	github.com/mattn/go-sqlite3 v1.14.22 // indirect

go.sum

@@ -1,15 +1,23 @@
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
+github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
 github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
 github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20240416155748-26353dc0451f h1:WpZiq8iqvGjJ3m3wzAVKL6+0vz7VkE79iSy9GII00II=
+github.com/google/pprof v0.0.0-20240416155748-26353dc0451f/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 h1:KwWnWVWCNtNq/ewIX7HIKnELmEx2nDP42yskD/pi7QE=
+github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
@@ -52,6 +60,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=

pkg/html/json-enc.js

@@ -0,0 +1,12 @@
+htmx.defineExtension('json-enc', {
+    onEvent: function (name, evt) {
+        if (name === "htmx:configRequest") {
+            evt.detail.headers['Content-Type'] = "application/json";
+        }
+    },
+    
+    encodeParameters : function(xhr, parameters, elt) {
+        xhr.overrideMimeType('text/json');
+        return (JSON.stringify(parameters));
+    }
+});

pkg/html/templates/home.html

@@ -13,6 +13,17 @@
                         <a href="/home" style="color: rgba(35, 207, 0, 0.87)">// Kyoketsu</a>
                     </div>
                 </div>
+                <div class="col-sm-8 p-3">
+                    <form method="post"
+                          hx-target="#response-div"
+                          hx-post="/refresh"
+                          hx-ext="json-enc">
+                          <input type="text" name="ip_address" placeholder="192.168.50.1/24" required>
+                          <button type="submit">send</button>
+                    </form>
+
+
+                </div>
             </div>
         </div>
         <div class="container-fluid row">
@@ -28,6 +39,7 @@
             <div class="col border border-white text-white"><p class="font-monospace fs-3">Ping Response?</p></div>
             <div class="col border border-white text-white"><p class="font-monospace fs-3">Listening Ports</p></div>
         </div>
+        <div id="response-div"></div>
         {{ range . }}
             {{ template "ip_table.html" . }}
         {{ end }}
@@ -35,6 +47,8 @@
     
     
         <script src="/static/htmx.min.js"></script>
+        <script src="/static/json-enc.js"></script>
+
     </body>
 
 

pkg/local.go

@@ -109,10 +109,7 @@ func addressRecurse(ipmap *IpSubnetMapper) {
 
 	next := getNextAddr(ipmap.Current.String())
 
-	nextNet := getNetwork(next, ipmap.Mask)
-	currentNet := ipmap.NetworkAddr.String()
-
-	if nextNet != currentNet {
+	if getNetwork(next, ipmap.Mask) != ipmap.NetworkAddr.String() {
 		return
 	}
 	ipmap.Current = net.ParseIP(next)

pkg/scanner.go

@@ -27,18 +27,12 @@ package kyoketsu
 import (
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 )
 
-var PORT_MAP = map[int]string{
-	22: "ssh", 23: "telnet", 53: "dns", 80: "http", 25: "smtp", 443: "https", 8080: "unknown", 8081: "unknown",
-	//8082: "unknown", 8085: "unknown", 8090: "unknown", 8091: "unknown", 9010: "unknown", 9012: "unknown", 10000: "unknown", 1433: "microsoft_sql",
-	3306: "mysql", 3050: "firebird", 5432: "postgres", 27017: "mongo", 6379: "redis", 8005: "tomcat", 6443: "kubernetes", 853: "dns-tls", 143: "imap",
-	389: "ldap", 445: "smb", 543: "kerberos", 544: "kerberos", 749: "kerberos", 760: "kerberos",
-}
-
 /*
 Need to work with with a database schema in mind, and revolve functionality around that
 */
@@ -58,31 +52,37 @@ Perform a concurrent TCP port dial on a host, either by domain name or IP.
 	:param addr: the address of fqdn to scan
 	:param portmap: a key/value pair of port numbers to service names to dial the host with
 */
-func PortWalk(addr string, portmap map[int]string) *Host {
+func PortWalk(addr string, portmap map[int]string) Host {
 	wg := &sync.WaitGroup{}
 	mu := &sync.Mutex{}
-	out := []*PortScanResult{}
+	out := map[int]string{}
 	for p, s := range portmap {
 		wg.Add(1)
 		go func(target string, p int, s string, mu *sync.Mutex) {
 			defer wg.Done()
 			scanout := singlePortScan(target, p, s)
-			mu.Lock()
-			out = append(out, scanout)
-			mu.Unlock()
+			if scanout.Listening {
+				mu.Lock()
+				out[scanout.PortNumber] = scanout.Service
+				mu.Unlock()
+			}
 		}(addr, p, s, mu)
 	}
 	wg.Wait()
-	host := &Host{IpAddress: addr, ListeningPorts: map[int]string{}}
-	for i := range out {
-		if out[i].Listening {
-			host.ListeningPorts[out[i].PortNumber] = out[i].Service
-			host.PortString = fmt.Sprintf("%s,%v", host.PortString, out[i].PortNumber)
-		}
-		host.PortString = strings.TrimPrefix(host.PortString, ",")
-
+	var dnames string
+	var portstring string
+	dns, _ := net.LookupAddr(addr)
+	dnames = strings.Join(dns, ", ")
+	for key, _ := range out {
+		portstring = portstring + "," + strconv.Itoa(key)
+	}
+	return Host{
+		IpAddress:      addr,
+		Fqdn:           dnames,
+		PingResponse:   false,
+		ListeningPorts: out,
+		PortString:     portstring,
 	}
-	return host
 
 }
 
@@ -94,17 +94,18 @@ type PortScanResult struct {
 	Listening  bool   `json:"listening"`   // A boolean value that depicts if the service is listening or not
 }
 
-type PortScanDirective struct {
-	// Struct for dependency injecting the dynamic port map used for scans
-	Pairs map[int]string
-}
-
 /*
 Wrapper function to dependency inject the resource for a port -> service name mapping.
 May move to a database, or something.
 */
-func RetrieveScanDirectives() PortScanDirective {
-	return PortScanDirective{Pairs: PORT_MAP}
+func RetrieveScanDirectives() map[int]string {
+	var portmap = map[int]string{
+		22: "ssh", 23: "telnet", 53: "dns", 80: "http", 25: "smtp", 443: "https", 8080: "unknown", 8081: "unknown",
+		8082: "unknown", 8085: "unknown", 8090: "unknown", 8091: "unknown", 9010: "unknown", 9012: "unknown", 10000: "unknown", 1433: "microsoft_sql",
+		3306: "mysql", 3050: "firebird", 5432: "postgres", 27017: "mongo", 6379: "redis", 8005: "tomcat", 6443: "kubernetes", 853: "dns-tls", 143: "imap",
+		389: "ldap", 445: "smb", 543: "kerberos", 544: "kerberos", 749: "kerberos", 760: "kerberos",
+	}
+	return portmap
 }
 
 /*
@@ -114,14 +115,13 @@ Scans a single host on a single port
 	:param port: the port number to dial
 	:param svcs: the name of the service that the port is associate with
 */
-func singlePortScan(addr string, port int, svcs string) *PortScanResult {
+func singlePortScan(addr string, port int, svcs string) PortScanResult {
 	address := fmt.Sprintf("%v:%d", addr, port)
-	conn, err := net.DialTimeout("tcp", address, 5*time.Second)
+	_, err := net.DialTimeout("tcp", address, 2*time.Second)
 	if err != nil {
-		return &PortScanResult{PortNumber: port, Protocol: "tcp", Service: svcs, Listening: false}
+		return PortScanResult{PortNumber: port, Protocol: "tcp", Service: svcs, Listening: false}
 	}
-	conn.Close()
-	return &PortScanResult{PortNumber: port, Protocol: "tcp", Service: svcs, Listening: true}
+	return PortScanResult{PortNumber: port, Protocol: "tcp", Service: svcs, Listening: true}
 }
 
 /*
@@ -130,32 +130,16 @@ Perform a port scan sweep across an entire subnet
 	:param ip: the IPv4 address WITH CIDR notation
 	:param portmap: the mapping of ports to scan with (port number mapped to protocol name)
 */
-func NetSweep(ip string, portmap map[int]string) ([]*Host, error) {
-	var err error
-	var addr *IpSubnetMapper
-	addr, err = GetNetworkAddresses(ip)
-	if err != nil {
-		return nil, err
-	}
-	returnhosts := []*Host{}
-
-	var wg sync.WaitGroup
-	mu := &sync.Mutex{}
-	for i := range addr.Ipv4s {
+func NetSweep(ips []net.IP, portmap map[int]string, scanned chan Host) {
+	wg := &sync.WaitGroup{}
+	for i := range ips {
 		wg.Add(1)
-		go func(target string, mu *sync.Mutex) {
+		go func(target string, wg *sync.WaitGroup) {
 			defer wg.Done()
-			out := PortWalk(target, portmap)
-			if len(out.ListeningPorts) > 0 {
-				dns, _ := net.LookupAddr(out.IpAddress)
-				out.Fqdn = strings.Join(dns, ", ")
-				mu.Lock()
-				returnhosts = append(returnhosts, out)
-				mu.Unlock()
-			}
-		}(addr.Ipv4s[i].String(), mu)
+			scanned <- PortWalk(target, portmap)
 
+		}(ips[i].String(), wg)
 	}
 	wg.Wait()
-	return returnhosts, nil
+
 }

pkg/storage.go

@@ -43,7 +43,7 @@ type TopologyDatabaseIO interface {
 	Migrate() error
 	Create(host Host) (*Host, error)
 	All() ([]Host, error)
-	GetByFqdn(dn string) (*Host, error)
+	GetByIP(ip string) (*Host, error)
 	Update(id int64, updated Host) (*Host, error)
 	Delete(id int64) error
 }
@@ -128,8 +128,8 @@ func (r *SQLiteRepo) All() ([]Host, error) {
 }
 
 // Get a record by its FQDN
-func (r *SQLiteRepo) GetByFqdn(dn string) (*Host, error) {
-	row := r.db.QueryRow("SELECT * FROM hosts WHERE fqdn = ?", dn)
+func (r *SQLiteRepo) GetByIP(ip string) (*Host, error) {
+	row := r.db.QueryRow("SELECT * FROM hosts WHERE ipv4_address = ?", ip)
 
 	var host Host
 	if err := row.Scan(&host.Id, &host.Fqdn, &host.IpAddress, &host.PortString); err != nil {
@@ -146,7 +146,7 @@ func (r *SQLiteRepo) Update(id int64, updated Host) (*Host, error) {
 	if id == 0 {
 		return nil, errors.New("invalid updated ID")
 	}
-	res, err := r.db.Exec("UPDATE hosts SET name = ?, url = ?, rank = ? WHERE id = ?", updated.Fqdn, updated.IpAddress, updated.PortString, id)
+	res, err := r.db.Exec("UPDATE hosts SET fqdn = ?, ipv4_address = ?, listening_port = ? WHERE id = ?", updated.Fqdn, updated.IpAddress, updated.PortString, id)
 	if err != nil {
 		return nil, err
 	}

pkg/webserver.go

@@ -2,17 +2,23 @@ package kyoketsu
 
 import (
 	"embed"
+	"encoding/json"
 	"fmt"
 	"html/template"
+	"io"
 	"log"
 	"net/http"
 	"path"
 	"strings"
 )
 
+type ScanRequest struct {
+	IpAddress string `json:"ip_address"`
+}
+
 // Holding all static web server resources
 //
-//go:embed html/bootstrap-5.0.2-dist/js/* html/bootstrap-5.0.2-dist/css/* html/htmx.min.js html/templates/* html/img/*
+//go:embed html/bootstrap-5.0.2-dist/js/* html/bootstrap-5.0.2-dist/css/* html/* html/templates/* html/img/*
 var content embed.FS
 
 /*
@@ -20,21 +26,68 @@ Run a new webserver
 
 	:param port: port number to run the webserver on
 */
-func RunHttpServer(port int, dbhook TopologyDatabaseIO) {
+func RunHttpServer(port int, dbhook TopologyDatabaseIO, portmap map[int]string) {
 	assets := &AssetHandler{Root: content, RelPath: "static", EmbedRoot: "html"}
 	tmpl, err := template.ParseFS(content, "html/templates/*.html")
 	if err != nil {
 		log.Fatal(err)
 	}
+	iptable, err := template.ParseFS(content, "html/templates/ip_table.html")
+	if err != nil {
+		log.Fatal(err)
+	}
 	htmlHndl := &HtmlHandler{Home: tmpl, DbHook: dbhook}
-
+	execHndl := &ExecutionHandler{DbHook: dbhook, PortMap: portmap, TableEntry: iptable}
 	http.Handle("/static/", assets)
 	http.Handle("/home", htmlHndl)
+	http.Handle("/refresh", execHndl)
 
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", port), nil))
 
 }
 
+type ExecutionHandler struct {
+	DbHook     TopologyDatabaseIO
+	TableEntry *template.Template
+	PortMap    map[int]string
+}
+
+func (e *ExecutionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	var input ScanRequest
+	b, err := io.ReadAll(r.Body)
+	defer r.Body.Close()
+	if err != nil {
+		fmt.Fprintf(w, "There was an error processing your request: %s", err)
+		return
+	}
+	err = json.Unmarshal(b, &input)
+	if err != nil {
+		fmt.Fprintf(w, "There was an error processing your request: %s", err)
+		return
+
+	}
+	subnetMap, err := GetNetworkAddresses(input.IpAddress)
+	if err != nil {
+		fmt.Fprintf(w, "There was an error processing your request: %s", err)
+	}
+	scanned := make(chan Host, len(subnetMap.Ipv4s))
+	NetSweep(subnetMap.Ipv4s, RetrieveScanDirectives(), scanned)
+	close(scanned)
+	for host := range scanned {
+		if len(host.ListeningPorts) > 0 {
+			e.TableEntry.Execute(w, host)
+			continue
+		} else if host.Fqdn != "" {
+			e.TableEntry.Execute(w, host)
+			continue
+		} else if host.PingResponse {
+			e.TableEntry.Execute(w, host)
+			continue
+		}
+	}
+
+}
+
 // handlers //
 
 type HtmlHandler struct {
@@ -52,7 +105,6 @@ func (h *HtmlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	data, err := h.DbHook.All()
 	if err != nil {
 		fmt.Fprintf(w, "You have made it to the kyoketsu web server!\nThere was an error getting the db table, though.\n%s", err)
-
 	}
 	h.Home.Execute(w, data)
 }