Added certificate based authentication option

Allow no auth connections
Added missing spacer between 'Uptime' and 'OS'
2024-12-13 00:21:17 -05:00 · 2024-12-09 15:30:18 -05:00 · 2024-12-06 12:31:57 -05:00 · 2024-12-06 12:21:56 -05:00 · 2024-12-05 22:15:32 -05:00 · 2024-12-02 22:36:12 -05:00
4 changed files with 453 additions and 275 deletions
@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+elastop

 # Test binary, built with `go test -c`
 *.test
@@ -32,12 +32,35 @@ go build
 ```

 ### Command Line Flags
-| Flag        | Description            | Default       |
-| ----------- | ---------------------- | ------------- |
-| `-host`     | Elasticsearch host     | `localhost`   |
-| `-port`     | Elasticsearch port     | `9200`        |
-| `-user`     | Elasticsearch username | `elastic`     |
-| `-password` | Elasticsearch password | `ES_PASSWORD` |
+| Flag        | Description                           | Default       |
+| ----------- | ------------------------------------- | ------------- |
+| `-host`     | Elasticsearch host                    | `localhost`   |
+| `-port`     | Elasticsearch port                    | `9200`        |
+| `-user`     | Elasticsearch username                | `elastic`     |
+| `-password` | Elasticsearch password                | `ES_PASSWORD` |
+| `-apikey`   | Elasticsearch API key                 | `ES_API_KEY`  |
+| `-cert`     | Path to client certificate file       |               |
+| `-key`      | Path to client private key file       |               |
+| `-ca`       | Path to CA certificate file           |               |
+| `-insecure` | Skip TLS certificate verification     | `false`       |
+
+Note: Only one authentication method (username/password, API key, or certificates) can be used at a time.
+
+### Authentication Examples
+
+```bash
+# Using username/password
+./elastop -host https://elasticsearch.example.com -user elastic -password secret
+
+# Using API key
+./elastop -host https://elasticsearch.example.com -apikey your_api_key
+
+# Using certificate authentication
+./elastop -host https://elasticsearch.example.com -cert /path/to/client.crt -key /path/to/client.key -ca /path/to/ca.crt
+
+# Using certificate authentication with insecure SSL (not recommended for production)
+./elastop -host https://elasticsearch.example.com -cert /path/to/client.crt -key /path/to/client.key -insecure
+```

 ## Dashboard Layout

@@ -2,6 +2,7 @@ package main

 import (
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"flag"
 	"fmt"
@@ -38,6 +39,19 @@ type ClusterStats struct {
 		Successful int `json:"successful"`
 		Failed     int `json:"failed"`
 	} `json:"_nodes"`
+	Process struct {
+		CPU struct {
+			Percent int `json:"percent"`
+		} `json:"cpu"`
+		OpenFileDescriptors struct {
+			Min int `json:"min"`
+			Max int `json:"max"`
+			Avg int `json:"avg"`
+		} `json:"open_file_descriptors"`
+	} `json:"process"`
+	Snapshots struct {
+		Count int `json:"count"`
+	} `json:"snapshots"`
 }

 type NodesInfo struct {
@@ -54,18 +68,8 @@ type NodesInfo struct {
 			PrettyName          string `json:"pretty_name"`
 		} `json:"os"`
 		Process struct {
-			ID       int  `json:"id"`
-			Mlockall bool `json:"mlockall"`
+			ID int `json:"id"`
 		} `json:"process"`
-		Settings struct {
-			Node struct {
-				Attr struct {
-					ML struct {
-						MachineMem string `json:"machine_memory"`
-					} `json:"ml"`
-				} `json:"attr"`
-			} `json:"node"`
-		} `json:"settings"`
 	} `json:"nodes"`
 }

@@ -80,7 +84,6 @@ type IndexStats []struct {

 type IndexActivity struct {
 	LastDocsCount    int
-	IsActive         bool
 	InitialDocsCount int
 	StartTime        time.Time
 }
@@ -153,6 +156,7 @@ type NodesStats struct {
 					} `json:"old"`
 				} `json:"collectors"`
 			} `json:"gc"`
+			UptimeInMillis int64 `json:"uptime_in_millis"`
 		} `json:"jvm"`
 		Transport struct {
 			RxSizeInBytes int64 `json:"rx_size_in_bytes"`
@@ -195,17 +199,6 @@ var (

 var indexActivities = make(map[string]*IndexActivity)

-type IngestionEvent struct {
-	Index     string
-	DocCount  int
-	Timestamp time.Time
-}
-
-type CatNodesStats struct {
-	Load1m string `json:"load_1m"`
-	Name   string `json:"name"`
-}
-
 var (
 	showNodes         = true
 	showRoles         = true
@@ -237,6 +230,11 @@ var (
 	apiKey string
 )

+type CatNodesStats struct {
+	Load1m string `json:"load_1m"`
+	Name   string `json:"name"`
+}
+
 func bytesToHuman(bytes int64) string {
 	const unit = 1024
 	if bytes < unit {
@@ -278,24 +276,6 @@ func convertSizeFormat(sizeStr string) string {
 	return fmt.Sprintf("%d%s", int(size), unit)
 }

-func formatResourceSize(bytes int64) string {
-	const unit = 1024
-	if bytes < unit {
-		return fmt.Sprintf("%4d B", bytes)
-	}
-
-	units := []string{"B", "K", "M", "G", "T", "P"}
-	exp := 0
-	val := float64(bytes)
-
-	for val >= unit && exp < len(units)-1 {
-		val /= unit
-		exp++
-	}
-
-	return fmt.Sprintf("%3d%s", int(val), units[exp])
-}
-
 func getPercentageColor(percent float64) string {
 	switch {
 	case percent < 30:
@@ -364,9 +344,9 @@ var roleColors = map[string]string{
 	"data_warm":             "#bd93f9", // purple
 	"data_cold":             "#f1fa8c", // yellow
 	"data_frozen":           "#ff79c6", // pink
-	"ingest":                "#87cefa", // light sky blue (was gray)
+	"ingest":                "#87cefa", // light sky blue
 	"ml":                    "#6272a4", // blue gray
-	"remote_cluster_client": "#dda0dd", // plum (was burgundy)
+	"remote_cluster_client": "#dda0dd", // plum
 	"transform":             "#689d6a", // forest green
 	"voting_only":           "#458588", // teal
 	"coordinating_only":     "#d65d0e", // burnt orange
@@ -389,6 +369,7 @@ var legendLabels = map[string]string{
 }

 func formatNodeRoles(roles []string) string {
+	// Define all possible roles and their letters in the desired order
 	roleMap := map[string]string{
 		"master":                "M",
 		"data":                  "D",
@@ -405,35 +386,38 @@ func formatNodeRoles(roles []string) string {
 		"coordinating_only":     "O",
 	}

-	// Get the role letters and sort them
-	var letters []string
+	// Create a map of the node's roles for quick lookup
+	nodeRoles := make(map[string]bool)
 	for _, role := range roles {
-		if letter, exists := roleMap[role]; exists {
-			letters = append(letters, letter)
-		}
-	}
-	sort.Strings(letters)
-
-	formattedRoles := "             "
-	runeRoles := []rune(formattedRoles)
-
-	for i, letter := range letters {
-		if i < 13 {
-			runeRoles[i] = []rune(letter)[0]
-		}
+		nodeRoles[role] = true
 	}

-	var result string
-	for _, r := range runeRoles {
-		if r == ' ' {
-			result += " "
+	// Create ordered list of role keys based on their letters
+	orderedRoles := []string{
+		"data_content",          // C
+		"data",                  // D
+		"data_frozen",           // F
+		"data_hot",              // H
+		"ingest",                // I
+		"data_cold",             // K
+		"ml",                    // L
+		"master",                // M
+		"coordinating_only",     // O
+		"remote_cluster_client", // R
+		"transform",             // T
+		"voting_only",           // V
+		"data_warm",             // W
+	}
+
+	result := ""
+	for _, role := range orderedRoles {
+		letter := roleMap[role]
+		if nodeRoles[role] {
+			// Node has this role - use the role's color
+			result += fmt.Sprintf("[%s]%s[white]", roleColors[role], letter)
 		} else {
-			for role, shortRole := range roleMap {
-				if string(r) == shortRole {
-					result += fmt.Sprintf("[%s]%s[white]", roleColors[role], string(r))
-					break
-				}
-			}
+			// Node doesn't have this role - use dark grey
+			result += fmt.Sprintf("[#444444]%s[white]", letter)
 		}
 	}

@@ -464,9 +448,6 @@ type indexInfo struct {
 	indexingRate float64
 }

-// Add startTime at package level
-var startTime = time.Now()
-
 func updateGridLayout(grid *tview.Grid, showRoles, showIndices, showMetrics bool) {
 	// Start with clean grid
 	grid.Clear()
@@ -482,7 +463,18 @@ func updateGridLayout(grid *tview.Grid, showRoles, showIndices, showMetrics bool
 		visiblePanels++
 	}

-	// Adjust row configuration based on whether nodes panel is shown
+	// When only nodes panel is visible, use a single column layout
+	if showNodes && visiblePanels == 0 {
+		grid.SetRows(3, 0) // Header and nodes only
+		grid.SetColumns(0) // Single full-width column
+
+		// Add header and nodes panel
+		grid.AddItem(header, 0, 0, 1, 1, 0, 0, false)
+		grid.AddItem(nodesPanel, 1, 0, 1, 1, 0, 0, false)
+		return
+	}
+
+	// Rest of the layout logic for when bottom panels are visible
 	if showNodes {
 		grid.SetRows(3, 0, 0) // Header, nodes, bottom panels
 	} else {
@@ -493,16 +485,16 @@ func updateGridLayout(grid *tview.Grid, showRoles, showIndices, showMetrics bool
 	switch {
 	case visiblePanels == 3:
 		if showRoles {
-			grid.SetColumns(30, -2, -1) // Changed from 20 to 30 for roles panel width
+			grid.SetColumns(30, -2, -1)
 		}
 	case visiblePanels == 2:
 		if showRoles {
-			grid.SetColumns(30, 0) // Changed from 20 to 30 for roles panel width
+			grid.SetColumns(30, 0)
 		} else {
-			grid.SetColumns(-1, -1) // Equal split between two panels
+			grid.SetColumns(-1, -1)
 		}
 	case visiblePanels == 1:
-		grid.SetColumns(0) // Single column takes full width
+		grid.SetColumns(0)
 	}

 	// Always show header at top spanning all columns
@@ -546,6 +538,13 @@ func main() {
 	user := flag.String("user", os.Getenv("ES_USER"), "Elasticsearch username")
 	password := flag.String("password", os.Getenv("ES_PASSWORD"), "Elasticsearch password")
 	flag.StringVar(&apiKey, "apikey", os.Getenv("ES_API_KEY"), "Elasticsearch API key")
+
+	// Add new certificate-related flags
+	certFile := flag.String("cert", "", "Path to client certificate file")
+	keyFile := flag.String("key", "", "Path to client private key file")
+	caFile := flag.String("ca", "", "Path to CA certificate file")
+	skipVerify := flag.Bool("insecure", false, "Skip TLS certificate verification")
+
 	flag.Parse()

 	// Validate and process the host URL
@@ -554,25 +553,65 @@ func main() {
 		os.Exit(1)
 	}

-	// Validate authentication
-	if apiKey != "" && (*user != "" || *password != "") {
-		fmt.Fprintf(os.Stderr, "Error: Cannot use both API key and username/password authentication\n")
+	// Validate authentication methods - only one should be used
+	authMethods := 0
+	if apiKey != "" {
+		authMethods++
+	}
+	if *user != "" || *password != "" {
+		authMethods++
+	}
+	if *certFile != "" || *keyFile != "" {
+		authMethods++
+	}
+	if authMethods > 1 {
+		fmt.Fprintf(os.Stderr, "Error: Cannot use multiple authentication methods simultaneously (API key, username/password, or certificates)\n")
 		os.Exit(1)
 	}

-	if apiKey == "" && (*user == "" || *password == "") {
-		fmt.Fprintf(os.Stderr, "Error: Must provide either API key or both username and password\n")
+	// Validate certificate files if specified
+	if (*certFile != "" && *keyFile == "") || (*certFile == "" && *keyFile != "") {
+		fmt.Fprintf(os.Stderr, "Error: Both certificate and key files must be specified together\n")
 		os.Exit(1)
 	}

 	// Strip any trailing slash from the host
 	*host = strings.TrimRight(*host, "/")

+	// Create TLS config
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: *skipVerify,
+	}
+
+	// Load client certificates if specified
+	if *certFile != "" && *keyFile != "" {
+		cert, err := tls.LoadX509KeyPair(*certFile, *keyFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error loading client certificates: %v\n", err)
+			os.Exit(1)
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+
+	// Load CA certificate if specified
+	if *caFile != "" {
+		caCert, err := os.ReadFile(*caFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error reading CA certificate: %v\n", err)
+			os.Exit(1)
+		}
+
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(caCert) {
+			fmt.Fprintf(os.Stderr, "Error parsing CA certificate\n")
+			os.Exit(1)
+		}
+		tlsConfig.RootCAs = caCertPool
+	}
+
 	// Create custom HTTP client with SSL configuration
 	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true, // Allow self-signed certificates
-		},
+		TLSClientConfig: tlsConfig,
 	}
 	client := &http.Client{
 		Transport: tr,
@@ -628,7 +667,7 @@ func main() {
 			// Set authentication
 			if apiKey != "" {
 				req.Header.Set("Authorization", fmt.Sprintf("ApiKey %s", apiKey))
-			} else {
+			} else if *user != "" && *password != "" {
 				req.SetBasicAuth(*user, *password)
 			}

@@ -692,35 +731,35 @@ func main() {
 			return
 		}

-		// Calculate aggregate metrics
+		// Query and indexing metrics
 		var (
-			totalQueries       int64
-			totalQueryTime     float64
-			totalIndexing      int64
-			totalIndexingTime  float64
-			totalCPUPercent    int
-			totalMemoryUsed    int64
-			totalMemoryTotal   int64
-			totalHeapUsed      int64
-			totalHeapMax       int64
-			totalGCCollections int64
-			totalGCTime        float64
-			nodeCount          int
+			totalQueries   int64
+			totalQueryTime int64
+			totalIndexing  int64
+			totalIndexTime int64
+			totalSegments  int64
 		)

 		for _, node := range nodesStats.Nodes {
 			totalQueries += node.Indices.Search.QueryTotal
-			totalQueryTime += float64(node.Indices.Search.QueryTimeInMillis) / 1000
+			totalQueryTime += node.Indices.Search.QueryTimeInMillis
 			totalIndexing += node.Indices.Indexing.IndexTotal
-			totalIndexingTime += float64(node.Indices.Indexing.IndexTimeInMillis) / 1000
-			totalCPUPercent += node.OS.CPU.Percent
-			totalMemoryUsed += node.OS.Memory.UsedInBytes
-			totalMemoryTotal += node.OS.Memory.TotalInBytes
-			totalHeapUsed += node.JVM.Memory.HeapUsedInBytes
-			totalHeapMax += node.JVM.Memory.HeapMaxInBytes
+			totalIndexTime += node.Indices.Indexing.IndexTimeInMillis
+			totalSegments += node.Indices.Segments.Count
+		}
+
+		queryRate := float64(totalQueries) / float64(totalQueryTime) * 1000  // queries per second
+		indexRate := float64(totalIndexing) / float64(totalIndexTime) * 1000 // docs per second
+
+		// GC metrics
+		var (
+			totalGCCollections int64
+			totalGCTime        int64
+		)
+
+		for _, node := range nodesStats.Nodes {
 			totalGCCollections += node.JVM.GC.Collectors.Young.CollectionCount + node.JVM.GC.Collectors.Old.CollectionCount
-			totalGCTime += float64(node.JVM.GC.Collectors.Young.CollectionTimeInMillis+node.JVM.GC.Collectors.Old.CollectionTimeInMillis) / 1000
-			nodeCount++
+			totalGCTime += node.JVM.GC.Collectors.Young.CollectionTimeInMillis + node.JVM.GC.Collectors.Old.CollectionTimeInMillis
 		}

 		// Update header
@@ -731,7 +770,7 @@ func main() {
 		}[clusterStats.Status]

 		// Get max lengths after fetching node and index info
-		maxNodeNameLen, maxIndexNameLen, maxTransportLen := getMaxLengths(nodesInfo, indicesStats)
+		maxNodeNameLen, maxIndexNameLen, maxTransportLen, maxIngestedLen := getMaxLengths(nodesInfo, indicesStats)

 		// Update header with dynamic padding
 		header.Clear()
@@ -812,25 +851,7 @@ func main() {
 				nodeLoads[node.Name] = node.Load1m
 			}

-			// In the update() function, add this request before processing nodes:
-			var threadPoolStats []ThreadPoolStats
-			if err := makeRequest("/_cat/thread_pool/generic?format=json&h=node_name,name,active,queue,rejected,completed", &threadPoolStats); err != nil {
-				nodesPanel.SetText(fmt.Sprintf("[red]Error getting thread pool stats: %v", err))
-				return
-			}
-
-			// Create a map for quick lookup of thread pool stats by node name
-			threadPoolMap := make(map[string]ThreadPoolStats)
-			for _, stat := range threadPoolStats {
-				threadPoolMap[stat.NodeName] = stat
-			}
-
-			active, _ := strconv.Atoi(threadPoolMap[nodeInfo.Name].Active)
-			queue, _ := strconv.Atoi(threadPoolMap[nodeInfo.Name].Queue)
-			rejected, _ := strconv.Atoi(threadPoolMap[nodeInfo.Name].Rejected)
-			completed, _ := strconv.Atoi(threadPoolMap[nodeInfo.Name].Completed)
-
-			fmt.Fprintf(nodesPanel, "[#5555ff]%-*s  [white] [#444444]│[white] %s [#444444]│[white] [white]%-*s[white] [#444444]│[white] [%s]%-7s[white] [#444444]│[white] [%s]%3d%% [#444444](%d)[white] [#444444]│[white] %4s [#444444]│[white] %4s / %4s [%s]%3d%%[white] [#444444]│[white] %4s / %4s [%s]%3d%%[white] [#444444]│[white] %4s / %4s [%s]%3d%%[white] [#444444]│[white] %6s [#444444]│[white] %5s [#444444]│[white] %8s [#444444]│[white] %9s [#444444]│[white] %s [#bd93f9]%s[white] [#444444](%s)[white]\n",
+			fmt.Fprintf(nodesPanel, "[#5555ff]%-*s [white] [#444444]│[white] %s [#444444]│[white] [white]%*s[white] [#444444]│[white] [%s]%-7s[white] [#444444]│[white] [%s]%3d%% [#444444](%d)[white] [#444444]│[white] %4s / %4s [%s]%3d%%[white] [#444444]│[white] %4s / %4s [%s]%3d%%[white] [#444444]│[white] %4s / %4s [%s]%3d%%[white] [#444444]│[white] %-8s[white] [#444444]│[white] %s [#bd93f9]%s[white] [#444444](%s)[white]\n",
 				maxNodeNameLen,
 				nodeInfo.Name,
 				formatNodeRoles(nodeInfo.Roles),
@@ -841,7 +862,6 @@ func main() {
 				getPercentageColor(float64(cpuPercent)),
 				cpuPercent,
 				nodeInfo.OS.AvailableProcessors,
-				nodeLoads[nodeInfo.Name],
 				formatResourceSize(nodeStats.OS.Memory.UsedInBytes),
 				formatResourceSize(nodeStats.OS.Memory.TotalInBytes),
 				getPercentageColor(memPercent),
@@ -854,10 +874,7 @@ func main() {
 				formatResourceSize(diskTotal),
 				getPercentageColor(diskPercent),
 				int(diskPercent),
-				formatNumber(active),
-				formatNumber(queue),
-				formatNumber(rejected),
-				formatNumber(completed),
+				formatUptime(nodeStats.JVM.UptimeInMillis),
 				nodeInfo.OS.PrettyName,
 				nodeInfo.OS.Version,
 				nodeInfo.OS.Arch)
@@ -873,7 +890,7 @@ func main() {
 		// Update indices panel with dynamic width
 		indicesPanel.Clear()
 		fmt.Fprintf(indicesPanel, "[::b][#00ffff][[#ff5555]4[#00ffff]] Indices Information[::-]\n\n")
-		fmt.Fprint(indicesPanel, getIndicesPanelHeader(maxIndexNameLen))
+		fmt.Fprint(indicesPanel, getIndicesPanelHeader(maxIndexNameLen, maxIngestedLen))

 		// Update index entries with dynamic width
 		var indices []indexInfo
@@ -932,8 +949,13 @@ func main() {
 			totalSize += node.FS.Total.TotalInBytes - node.FS.Total.AvailableInBytes
 		}

-		// Sort indices by document count (descending)
+		// Sort indices - active ones first, then alphabetically within each group
 		sort.Slice(indices, func(i, j int) bool {
+			// If one is active and the other isn't, active goes first
+			if (indices[i].indexingRate > 0) != (indices[j].indexingRate > 0) {
+				return indices[i].indexingRate > 0
+			}
+			// Within the same group (both active or both inactive), sort alphabetically
 			return indices[i].index < indices[j].index
 		})

@@ -947,17 +969,17 @@ func main() {
 			// Add data stream indicator
 			streamIndicator := " "
 			if isDataStream(idx.index, dataStreamResp) {
-				streamIndicator = "[#bd93f9]⚡[white]"
+				streamIndicator = "[#bd93f9]⚫[white]"
 			}

-			// Calculate document changes
+			// Calculate document changes with dynamic padding
 			activity := indexActivities[idx.index]
 			ingestedStr := ""
 			if activity != nil && activity.InitialDocsCount < idx.docs {
 				docChange := idx.docs - activity.InitialDocsCount
-				ingestedStr = fmt.Sprintf("[green]%-12s", fmt.Sprintf("+%s", formatNumber(docChange)))
+				ingestedStr = fmt.Sprintf("[green]%-*s", maxIngestedLen, fmt.Sprintf("+%s", formatNumber(docChange)))
 			} else {
-				ingestedStr = fmt.Sprintf("%-12s", "")
+				ingestedStr = fmt.Sprintf("%-*s", maxIngestedLen, "")
 			}

 			// Format indexing rate
@@ -975,7 +997,7 @@ func main() {
 			// Convert the size format before display
 			sizeStr := convertSizeFormat(idx.storeSize)

-			fmt.Fprintf(indicesPanel, "%s %s[%s]%-*s[white] [#444444]│[white] %15s [#444444]│[white] %12s [#444444]│[white] %8s [#444444]│[white] %8s [#444444]│[white] %s [#444444]│[white] %-8s\n",
+			fmt.Fprintf(indicesPanel, "%s %s[%s]%-*s[white] [#444444]│[white] %13s [#444444]│[white] %5s [#444444]│[white] %6s [#444444]│[white] %8s [#444444]│[white] %-*s [#444444]│[white] %-8s\n",
 				writeIcon,
 				streamIndicator,
 				getHealthColor(idx.health),
@@ -985,6 +1007,7 @@ func main() {
 				sizeStr,
 				idx.priShards,
 				idx.replicas,
+				maxIngestedLen,
 				ingestedStr,
 				rateStr)
 		}
@@ -1028,109 +1051,106 @@ func main() {
 		metricsPanel.Clear()
 		fmt.Fprintf(metricsPanel, "[::b][#00ffff][[#ff5555]5[#00ffff]] Cluster Metrics[::-]\n\n")

-		// Helper function to format metric lines with consistent alignment
-		formatMetric := func(name string, value string) string {
-			return fmt.Sprintf("[#00ffff]%-25s[white] %s\n", name+":", value)
+		// Define metrics keys with proper grouping
+		metricKeys := []string{
+			// System metrics
+			"CPU",
+			"Memory",
+			"Heap",
+			"Disk",
+
+			// Network metrics
+			"Network TX",
+			"Network RX",
+			"HTTP Connections",
+
+			// Performance metrics
+			"Query Rate",
+			"Index Rate",
+
+			// Miscellaneous
+			"Snapshots",
 		}

-		// Search metrics
-		fmt.Fprint(metricsPanel, formatMetric("Search Queries", formatNumber(int(totalQueries))))
-		fmt.Fprint(metricsPanel, formatMetric("Query Rate", fmt.Sprintf("%s/s", formatNumber(int(float64(totalQueries)/time.Since(startTime).Seconds())))))
-		fmt.Fprint(metricsPanel, formatMetric("Total Query Time", fmt.Sprintf("%.1fs", totalQueryTime)))
-		fmt.Fprint(metricsPanel, formatMetric("Avg Query Latency", fmt.Sprintf("%.2fms", totalQueryTime*1000/float64(totalQueries+1))))
+		// Find the longest key for proper alignment
+		maxKeyLength := 0
+		for _, key := range metricKeys {
+			if len(key) > maxKeyLength {
+				maxKeyLength = len(key)
+			}
+		}

-		// Indexing metrics
-		fmt.Fprint(metricsPanel, formatMetric("Index Operations", formatNumber(int(totalIndexing))))
-		fmt.Fprint(metricsPanel, formatMetric("Indexing Rate", fmt.Sprintf("%s/s", formatNumber(int(float64(totalIndexing)/time.Since(startTime).Seconds())))))
-		fmt.Fprint(metricsPanel, formatMetric("Total Index Time", fmt.Sprintf("%.1fs", totalIndexingTime)))
-		fmt.Fprint(metricsPanel, formatMetric("Avg Index Latency", fmt.Sprintf("%.2fms", totalIndexingTime*1000/float64(totalIndexing+1))))
+		// Add padding for better visual separation
+		maxKeyLength += 2

-		// GC metrics
-		fmt.Fprint(metricsPanel, formatMetric("GC Collections", formatNumber(int(totalGCCollections))))
-		fmt.Fprint(metricsPanel, formatMetric("Total GC Time", fmt.Sprintf("%.1fs", totalGCTime)))
-		fmt.Fprint(metricsPanel, formatMetric("Avg GC Time", fmt.Sprintf("%.2fms", totalGCTime*1000/float64(totalGCCollections+1))))
+		// Helper function for metric lines with proper alignment
+		formatMetric := func(name string, value string) string {
+			return fmt.Sprintf("[#00ffff]%-*s[white] %s\n", maxKeyLength, name+":", value)
+		}
+
+		// CPU metrics
+		totalProcessors := 0
+		for _, node := range nodesInfo.Nodes {
+			totalProcessors += node.OS.AvailableProcessors
+		}
+		cpuPercent := float64(clusterStats.Process.CPU.Percent)
+		fmt.Fprint(metricsPanel, formatMetric("CPU", fmt.Sprintf("%7.1f%% [#444444](%d processors)[white]", cpuPercent, totalProcessors)))
+
+		// Disk metrics
+		diskUsed := getTotalSize(nodesStats)
+		diskTotal := getTotalDiskSpace(nodesStats)
+		diskPercent := float64(diskUsed) / float64(diskTotal) * 100
+		fmt.Fprint(metricsPanel, formatMetric("Disk", fmt.Sprintf("%8s / %8s [%s]%5.1f%%[white]",
+			bytesToHuman(diskUsed),
+			bytesToHuman(diskTotal),
+			getPercentageColor(diskPercent),
+			diskPercent)))
+
+		// Calculate heap and memory totals
+		var (
+			totalHeapUsed    int64
+			totalHeapMax     int64
+			totalMemoryUsed  int64
+			totalMemoryTotal int64
+		)
+
+		for _, node := range nodesStats.Nodes {
+			totalHeapUsed += node.JVM.Memory.HeapUsedInBytes
+			totalHeapMax += node.JVM.Memory.HeapMaxInBytes
+			totalMemoryUsed += node.OS.Memory.UsedInBytes
+			totalMemoryTotal += node.OS.Memory.TotalInBytes
+		}
+
+		// Heap metrics
+		heapPercent := float64(totalHeapUsed) / float64(totalHeapMax) * 100
+		fmt.Fprint(metricsPanel, formatMetric("Heap", fmt.Sprintf("%8s / %8s [%s]%5.1f%%[white]",
+			bytesToHuman(totalHeapUsed),
+			bytesToHuman(totalHeapMax),
+			getPercentageColor(heapPercent),
+			heapPercent)))

 		// Memory metrics
-		totalMemoryPercent := float64(totalMemoryUsed) / float64(totalMemoryTotal) * 100
-		totalHeapPercent := float64(totalHeapUsed) / float64(totalHeapMax) * 100
-		fmt.Fprint(metricsPanel, formatMetric("Memory Usage", fmt.Sprintf("%s / %s (%.1f%%)", bytesToHuman(totalMemoryUsed), bytesToHuman(totalMemoryTotal), totalMemoryPercent)))
-		fmt.Fprint(metricsPanel, formatMetric("Heap Usage", fmt.Sprintf("%s / %s (%.1f%%)", bytesToHuman(totalHeapUsed), bytesToHuman(totalHeapMax), totalHeapPercent)))
-
-		// Segment metrics
-		fmt.Fprint(metricsPanel, formatMetric("Total Segments", formatNumber(int(getTotalSegments(nodesStats)))))
-		fmt.Fprint(metricsPanel, formatMetric("Open File Descriptors", formatNumber(int(getTotalOpenFiles(nodesStats)))))
+		memoryPercent := float64(totalMemoryUsed) / float64(totalMemoryTotal) * 100
+		fmt.Fprint(metricsPanel, formatMetric("Memory", fmt.Sprintf("%8s / %8s [%s]%5.1f%%[white]",
+			bytesToHuman(totalMemoryUsed),
+			bytesToHuman(totalMemoryTotal),
+			getPercentageColor(memoryPercent),
+			memoryPercent)))

 		// Network metrics
-		fmt.Fprint(metricsPanel, formatMetric("Network TX", bytesToHuman(getTotalNetworkTX(nodesStats))))
-		fmt.Fprint(metricsPanel, formatMetric("Network RX", bytesToHuman(getTotalNetworkRX(nodesStats))))
+		fmt.Fprint(metricsPanel, formatMetric("Network TX", fmt.Sprintf(" %7s", bytesToHuman(getTotalNetworkTX(nodesStats)))))
+		fmt.Fprint(metricsPanel, formatMetric("Network RX", fmt.Sprintf(" %7s", bytesToHuman(getTotalNetworkRX(nodesStats)))))

-		// Disk I/O metrics
-		totalDiskReads := int64(0)
-		totalDiskWrites := int64(0)
-		for _, node := range nodesStats.Nodes {
-			totalDiskReads += node.FS.DiskReads
-			totalDiskWrites += node.FS.DiskWrites
-		}
-		fmt.Fprint(metricsPanel, formatMetric("Disk Reads", formatNumber(int(totalDiskReads))))
-		fmt.Fprint(metricsPanel, formatMetric("Disk Writes", formatNumber(int(totalDiskWrites))))
+		// HTTP Connections and Shard metrics - right aligned to match Network RX 'G'
+		fmt.Fprint(metricsPanel, formatMetric("HTTP Connections", fmt.Sprintf("%8s", formatNumber(int(getTotalHTTPConnections(nodesStats))))))
+		fmt.Fprint(metricsPanel, formatMetric("Query Rate", fmt.Sprintf("%6s/s", formatNumber(int(queryRate)))))
+		fmt.Fprint(metricsPanel, formatMetric("Index Rate", fmt.Sprintf("%6s/s", formatNumber(int(indexRate)))))

-		// HTTP connections
-		totalHTTPConnections := int64(0)
-		for _, node := range nodesStats.Nodes {
-			totalHTTPConnections += node.HTTP.CurrentOpen
-		}
-		fmt.Fprint(metricsPanel, formatMetric("HTTP Connections", formatNumber(int(totalHTTPConnections))))
+		// Snapshots
+		fmt.Fprint(metricsPanel, formatMetric("Snapshots", fmt.Sprintf("%8s", formatNumber(clusterStats.Snapshots.Count))))

-		// Average CPU usage across nodes
-		avgCPUPercent := float64(totalCPUPercent) / float64(nodeCount)
-		fmt.Fprint(metricsPanel, formatMetric("Average CPU Usage", fmt.Sprintf("%.1f%%", avgCPUPercent)))
-
-		// Pending tasks
-		fmt.Fprint(metricsPanel, formatMetric("Pending Tasks", formatNumber(clusterHealth.NumberOfPendingTasks)))
-		if clusterHealth.TaskMaxWaitingTime != "" && clusterHealth.TaskMaxWaitingTime != "0s" {
-			fmt.Fprint(metricsPanel, formatMetric("Max Task Wait Time", clusterHealth.TaskMaxWaitingTime))
-		}
-
-		// Update roles panel
-		rolesPanel.Clear()
-		fmt.Fprintf(rolesPanel, "[::b][#00ffff][[#ff5555]3[#00ffff]] Node Roles[::-]\n\n")
-
-		// Create a map of used roles
-		usedRoles := make(map[string]bool)
-		for _, nodeInfo := range nodesInfo.Nodes {
-			for _, role := range nodeInfo.Roles {
-				usedRoles[role] = true
-			}
-		}
-
-		// Display roles in the roles panel
-		roleLegend := [][2]string{
-			{"C", "data_content"},
-			{"D", "data"},
-			{"F", "data_frozen"},
-			{"H", "data_hot"},
-			{"I", "ingest"},
-			{"K", "data_cold"},
-			{"L", "ml"},
-			{"M", "master"},
-			{"O", "coordinating_only"},
-			{"R", "remote_cluster_client"},
-			{"T", "transform"},
-			{"V", "voting_only"},
-			{"W", "data_warm"},
-		}
-
-		for _, role := range roleLegend {
-			if usedRoles[role[1]] {
-				fmt.Fprintf(rolesPanel, "[%s]%s[white] %s\n",
-					roleColors[role[1]],
-					role[0],
-					legendLabels[role[1]])
-			} else {
-				fmt.Fprintf(rolesPanel, "[#444444]%s %s\n",
-					role[0],
-					legendLabels[role[1]])
-			}
+		if showRoles {
+			updateRolesPanel(rolesPanel, nodesInfo)
 		}
 	}

@@ -1178,22 +1198,6 @@ func main() {
 	}
 }

-func getTotalSegments(stats NodesStats) int64 {
-	var total int64
-	for _, node := range stats.Nodes {
-		total += node.Indices.Segments.Count
-	}
-	return total
-}
-
-func getTotalOpenFiles(stats NodesStats) int64 {
-	var total int64
-	for _, node := range stats.Nodes {
-		total += node.Process.OpenFileDescriptors
-	}
-	return total
-}
-
 func getTotalNetworkTX(stats NodesStats) int64 {
 	var total int64
 	for _, node := range stats.Nodes {
@@ -1210,10 +1214,11 @@ func getTotalNetworkRX(stats NodesStats) int64 {
 	return total
 }

-func getMaxLengths(nodesInfo NodesInfo, indicesStats IndexStats) (int, int, int) {
+func getMaxLengths(nodesInfo NodesInfo, indicesStats IndexStats) (int, int, int, int) {
 	maxNodeNameLen := 0
 	maxIndexNameLen := 0
 	maxTransportLen := 0
+	maxIngestedLen := 8 // Start with "Ingested" header length

 	// Get max node name and transport address length
 	for _, nodeInfo := range nodesInfo.Nodes {
@@ -1225,26 +1230,38 @@ func getMaxLengths(nodesInfo NodesInfo, indicesStats IndexStats) (int, int, int)
 		}
 	}

-	// Get max index name length only for visible indices
+	// Get max index name length and calculate max ingested length
 	for _, index := range indicesStats {
-		// Only consider indices that should be visible based on showHiddenIndices
 		if (showHiddenIndices || !strings.HasPrefix(index.Index, ".")) && index.DocsCount != "0" {
 			if len(index.Index) > maxIndexNameLen {
 				maxIndexNameLen = len(index.Index)
 			}
+
+			docs := 0
+			fmt.Sscanf(index.DocsCount, "%d", &docs)
+			if activity := indexActivities[index.Index]; activity != nil {
+				if activity.InitialDocsCount < docs {
+					docChange := docs - activity.InitialDocsCount
+					ingestedStr := fmt.Sprintf("+%s", formatNumber(docChange))
+					if len(ingestedStr) > maxIngestedLen {
+						maxIngestedLen = len(ingestedStr)
+					}
+				}
+			}
 		}
 	}

 	// Add padding
 	maxNodeNameLen += 2
-	maxIndexNameLen += 1 // Single space before separator
-	maxTransportLen += 2 // Add some padding for transport address
+	maxIndexNameLen += 1 // Changed from 2 to 1 for minimal padding
+	maxTransportLen += 2
+	maxIngestedLen += 1 // Minimal padding for ingested column

-	return maxNodeNameLen, maxIndexNameLen, maxTransportLen
+	return maxNodeNameLen, maxIndexNameLen, maxTransportLen, maxIngestedLen
 }

 func getNodesPanelHeader(maxNodeNameLen, maxTransportLen int) string {
-	return fmt.Sprintf("[::b]%-*s  [#444444]│[#00ffff] %-13s [#444444]│[#00ffff] %-*s [#444444]│[#00ffff] %-7s [#444444]│[#00ffff] %4s      [#444444]│[#00ffff] %4s [#444444]│[#00ffff] %-16s [#444444]│[#00ffff] %-16s [#444444]│[#00ffff] %-16s [#444444]│[#00ffff] %6s [#444444]│[#00ffff] %5s [#444444]│[#00ffff] %8s [#444444]│[#00ffff] %9s [#444444]│[#00ffff] %-25s[white]\n",
+	return fmt.Sprintf("[::b]%-*s [#444444]│[#00ffff] %-13s [#444444]│[#00ffff] %*s [#444444]│[#00ffff] %-7s [#444444]│[#00ffff] %-9s [#444444]│[#00ffff] %-16s [#444444]│[#00ffff] %-16s [#444444]│[#00ffff] %-16s [#444444]│[#00ffff] %-6s [#444444]│[#00ffff] %-25s[white]\n",
 		maxNodeNameLen,
 		"Node Name",
 		"Roles",
@@ -1252,25 +1269,22 @@ func getNodesPanelHeader(maxNodeNameLen, maxTransportLen int) string {
 		"Transport Address",
 		"Version",
 		"CPU",
-		"Load",
 		"Memory",
 		"Heap",
 		"Disk",
-		"Active",
-		"Queue",
-		"Rejected",
-		"Completed",
+		"Uptime",
 		"OS")
 }

-func getIndicesPanelHeader(maxIndexNameLen int) string {
-	return fmt.Sprintf("   [::b] %-*s [#444444]│[#00ffff] %15s [#444444]│[#00ffff] %12s [#444444]│[#00ffff] %8s [#444444]│[#00ffff] %8s [#444444]│[#00ffff] %-12s [#444444]│[#00ffff] %-8s[white]\n",
+func getIndicesPanelHeader(maxIndexNameLen, maxIngestedLen int) string {
+	return fmt.Sprintf("   [::b] %-*s [#444444]│[#00ffff] %13s [#444444]│[#00ffff] %5s [#444444]│[#00ffff] %6s [#444444]│[#00ffff] %8s [#444444]│[#00ffff] %-*s [#444444][#00ffff] %-8s[white]\n",
 		maxIndexNameLen,
 		"Index Name",
 		"Documents",
 		"Size",
 		"Shards",
 		"Replicas",
+		maxIngestedLen,
 		"Ingested",
 		"Rate")
 }
@@ -1284,12 +1298,152 @@ func isDataStream(name string, dataStreams DataStreamResponse) bool {
 	return false
 }

-// Add this with the other type definitions near the top of the file
-type ThreadPoolStats struct {
-	NodeName  string `json:"node_name"`
-	Name      string `json:"name"`
-	Active    string `json:"active"`
-	Queue     string `json:"queue"`
-	Rejected  string `json:"rejected"`
-	Completed string `json:"completed"`
+func getTotalSize(stats NodesStats) int64 {
+	var total int64
+	for _, node := range stats.Nodes {
+		if len(node.FS.Data) > 0 {
+			total += node.FS.Data[0].TotalInBytes - node.FS.Data[0].AvailableInBytes
+		}
+	}
+	return total
+}
+
+func getTotalDiskSpace(stats NodesStats) int64 {
+	var total int64
+	for _, node := range stats.Nodes {
+		if len(node.FS.Data) > 0 {
+			total += node.FS.Data[0].TotalInBytes
+		}
+	}
+	return total
+}
+
+func formatUptime(uptimeMillis int64) string {
+	uptime := time.Duration(uptimeMillis) * time.Millisecond
+	days := int(uptime.Hours() / 24)
+	hours := int(uptime.Hours()) % 24
+	minutes := int(uptime.Minutes()) % 60
+
+	var result string
+	if days > 0 {
+		result = fmt.Sprintf("%d[#ff99cc]d[white]%d[#ff99cc]h[white]", days, hours)
+	} else if hours > 0 {
+		result = fmt.Sprintf("%d[#ff99cc]h[white]%d[#ff99cc]m[white]", hours, minutes)
+	} else {
+		result = fmt.Sprintf("%d[#ff99cc]m[white]", minutes)
+	}
+
+	// Calculate the actual display length by removing all color codes in one pass
+	displayLen := len(strings.NewReplacer(
+		"[#ff99cc]", "",
+		"[white]", "",
+	).Replace(result))
+
+	// Add padding to make all uptime strings align (6 chars for display)
+	padding := 6 - displayLen
+	if padding > 0 {
+		result = strings.TrimRight(result, " ") + strings.Repeat(" ", padding)
+	}
+
+	return result
+}
+
+func getTotalHTTPConnections(stats NodesStats) int64 {
+	var total int64
+	for _, node := range stats.Nodes {
+		total += node.HTTP.CurrentOpen
+	}
+	return total
+}
+
+func updateRolesPanel(rolesPanel *tview.TextView, nodesInfo NodesInfo) {
+	rolesPanel.Clear()
+	fmt.Fprintf(rolesPanel, "[::b][#00ffff][[#ff5555]3[#00ffff]] Legend[::-]\n\n")
+
+	// Add Node Roles title in cyan
+	fmt.Fprintf(rolesPanel, "[::b][#00ffff]Node Roles[::-]\n")
+
+	// Define role letters (same as in formatNodeRoles)
+	roleMap := map[string]string{
+		"master":                "M",
+		"data":                  "D",
+		"data_content":          "C",
+		"data_hot":              "H",
+		"data_warm":             "W",
+		"data_cold":             "K",
+		"data_frozen":           "F",
+		"ingest":                "I",
+		"ml":                    "L",
+		"remote_cluster_client": "R",
+		"transform":             "T",
+		"voting_only":           "V",
+		"coordinating_only":     "O",
+	}
+
+	// Create a map of active roles in the cluster
+	activeRoles := make(map[string]bool)
+	for _, node := range nodesInfo.Nodes {
+		for _, role := range node.Roles {
+			activeRoles[role] = true
+		}
+	}
+
+	// Sort roles alphabetically by their letters
+	var roles []string
+	for role := range legendLabels {
+		roles = append(roles, role)
+	}
+	sort.Slice(roles, func(i, j int) bool {
+		return roleMap[roles[i]] < roleMap[roles[j]]
+	})
+
+	// Display each role with its color and description
+	for _, role := range roles {
+		color := roleColors[role]
+		label := legendLabels[role]
+		letter := roleMap[role]
+
+		// If role is not active in cluster, use grey color for the label
+		labelColor := "[white]"
+		if !activeRoles[role] {
+			labelColor = "[#444444]"
+		}
+
+		fmt.Fprintf(rolesPanel, "[%s]%s[white] %s%s\n", color, letter, labelColor, label)
+	}
+
+	// Add version status information
+	fmt.Fprintf(rolesPanel, "\n[::b][#00ffff]Version Status[::-]\n")
+	fmt.Fprintf(rolesPanel, "[green]⚫[white] Up to date\n")
+	fmt.Fprintf(rolesPanel, "[yellow]⚫[white] Outdated\n")
+
+	// Add index health status information
+	fmt.Fprintf(rolesPanel, "\n[::b][#00ffff]Index Health[::-]\n")
+	fmt.Fprintf(rolesPanel, "[green]⚫[white] All shards allocated\n")
+	fmt.Fprintf(rolesPanel, "[#ffff00]⚫[white] Replica shards unallocated\n")
+	fmt.Fprintf(rolesPanel, "[#ff5555]⚫[white] Primary shards unallocated\n")
+
+	// Add index status indicators
+	fmt.Fprintf(rolesPanel, "\n[::b][#00ffff]Index Status[::-]\n")
+	fmt.Fprintf(rolesPanel, "[#5555ff]⚫[white] Active indexing\n")
+	fmt.Fprintf(rolesPanel, "[#444444]⚪[white] No indexing\n")
+	fmt.Fprintf(rolesPanel, "[#bd93f9]⚫[white] Data stream\n")
+}
+
+func formatResourceSize(bytes int64) string {
+	const unit = 1024
+	if bytes < unit {
+		return fmt.Sprintf("%4d B", bytes)
+	}
+
+	units := []string{"B", "K", "M", "G", "T", "P"}
+	exp := 0
+	val := float64(bytes)
+
+	for val >= unit && exp < len(units)-1 {
+		val /= unit
+		exp++
+	}
+
+	return fmt.Sprintf("%3d%s", int(val), units[exp])
 }
Author	SHA1	Message	Date
acidvegas	273fed1776	Added certificate based authentication option	2024-12-13 00:21:17 -05:00
acidvegas	4bbf0c6bf1	Allow no auth connections	2024-12-09 15:30:18 -05:00
acidvegas	23a7c216ab	Added missing spacer between 'Uptime' and 'OS'	2024-12-06 12:31:57 -05:00
acidvegas	66e5e4a1b3	Fixed more padding & layout issues, renamed 'Node Roles' to 'Legend', cleaned up cluster metrics, etc	2024-12-06 12:21:56 -05:00
acidvegas	0912d77018	Added node uptimes, fixed padding and spacing in various spots to fit everything better. cluster stats box being redone with better stat displays (still working on that)	2024-12-05 22:15:32 -05:00
acidvegas	1d08518847	Indexes being written to are sorted to the top now	2024-12-02 22:36:12 -05:00