Documentation improved with -j (stdout jsonl lines) and -debug information about CAIDA style error addresses

README.md

@@ -14,6 +14,7 @@ PTRStream is a fast and efficient PTR record scanner designed for distributed sc
 - Automatic DNS server rotation from public resolvers
 - Progress tracking with detailed statistics
 - Colorized terminal output
+- CAIDA-style error formatting (with -debug flag)
 
 ## Installation
 
@@ -27,6 +28,7 @@ go install github.com/acidvegas/ptrstream@latest
 | `-c`     | `int`    | `100`   | Concurrency level                          |
 | `-debug` | `bool`   | `false` | Show unsuccessful lookups                  |
 | `-dns`   | `string` |         | File containing DNS servers                |
+| `-j`     | `bool`   | `false` | Output NDJSON to stdout (no TUI)          |
 | `-l`     | `bool`   | `false` | Loop continuously after completion         |
 | `-o`     | `string` |         | Path to NDJSON output file                 |
 | `-r`     | `int`    | `2`     | Number of retries for failed lookups       |
@@ -34,7 +36,6 @@ go install github.com/acidvegas/ptrstream@latest
 | `-shard` | `string` |         | Shard specification *(index/total format)* |
 | `-t`     | `int`    | `2`     | Timeout for DNS queries                    |
 
-
 ## Usage
 
 ```bash
@@ -114,6 +115,30 @@ Example NDJSON output:
 {"timestamp":"2024-01-05T12:34:56Z","ip_addr":"1.2.3.4","dns_server":"8.8.8.8","ptr_record":"example.com","record_type":"PTR","ttl":3600}
 {"timestamp":"2024-01-05T12:34:57Z","ip_addr":"5.6.7.8","dns_server":"1.1.1.1","ptr_record":"original.com","record_type":"CNAME","target":"target.com","ttl":600}
 ```
----
+
+## Debug Mode
+
+When running with `-debug`, failed lookups are displayed and logged using CAIDA-style error formatting. Each error is represented as a special `.in-addr.arpa` address:
+
+```
+2024-01-05 12:34:56 │ 1.2.3.4     │ 8.8.8.8        │  ERR  │        │ FAIL.TIMEOUT.in-addr.arpa
+2024-01-05 12:34:57 │ 5.6.7.8     │ 1.1.1.1        │  ERR  │        │ FAIL.SERVER-FAILURE.in-addr.arpa
+2024-01-05 12:34:58 │ 9.10.11.12  │ 8.8.4.4        │  ERR  │        │ FAIL.NON-AUTHORITATIVE.in-addr.arpa
+```
+
+Error types include:
+- `FAIL.TIMEOUT.in-addr.arpa` - DNS query timed out
+- `FAIL.SERVER-FAILURE.in-addr.arpa` - DNS server returned an error
+- `FAIL.NON-AUTHORITATIVE.in-addr.arpa` - No authoritative answer
+- `FAIL.REFUSED.in-addr.arpa` - Query was refused
+- `FAIL.NO-PTR-RECORD.in-addr.arpa` - No PTR record exists
+- And more...
+
+These errors are also included in the NDJSON output when using `-debug` with either `-o` or `-j`:
+```json
+{"seen":"2024-01-05T12:34:56Z","ip":"1.2.3.4","nameserver":"8.8.8.8","record":"FAIL.TIMEOUT.in-addr.arpa","record_type":"ERR","ttl":0}
+```
+
+___
 
 ###### Mirrors: [acid.vegas](https://git.acid.vegas/ptrstream) • [SuperNETs](https://git.supernets.org/acidvegas/ptrstream) • [GitHub](https://github.com/acidvegas/ptrstream) • [GitLab](https://gitlab.com/acidvegas/ptrstream) • [Codeberg](https://codeberg.org/acidvegas/ptrstream)

ptrstream.go

@@ -47,6 +47,7 @@ type Stats struct {
 	cnames        uint64
 	speedHistory  []float64
 	mu            sync.Mutex
+	round         uint64
 }
 
 func (s *Stats) increment() {
@@ -359,21 +360,21 @@ func worker(jobs <-chan string, wg *sync.WaitGroup, cfg *Config, stats *Stats, t
 		if err != nil {
 			stats.incrementFailed()
 			if cfg.debug {
-				errMsg := err.Error()
-				if idx := strings.LastIndex(errMsg, ": "); idx != -1 {
-					errMsg = errMsg[idx+2:]
-				}
+				errRecord := formatErrorAsHostname(err)
 				timeStr := time.Now().Format("2006-01-02 15:04:05")
 				line := fmt.Sprintf("[gray]%s [gray]│[-] [purple]%15s[-] [gray]│[-] [aqua]%-15s[-] [gray]│[-] [red] ERR [-] [gray]│[-] [gray]%-6s[-] [gray]│[-] [gray]%s[-]\n",
 					timeStr,
 					ip,
 					server,
 					"",
-					errMsg)
+					errRecord)
 				app.QueueUpdateDraw(func() {
 					fmt.Fprint(textView, line)
 					textView.ScrollToEnd()
 				})
+
+				// Write to NDJSON if enabled
+				writeNDJSON(cfg, time.Now(), ip, server, errRecord, "ERR", "", 0)
 			}
 			continue
 		}
@@ -400,7 +401,7 @@ func worker(jobs <-chan string, wg *sync.WaitGroup, cfg *Config, stats *Stats, t
 		ptr := ""
 		for _, name := range response.Names {
 			if cleaned := strings.TrimSpace(strings.TrimSuffix(name, ".")); cleaned != "" {
-				ptr = cleaned
+				ptr = strings.ToLower(cleaned)
 				break
 			}
 		}
@@ -416,7 +417,7 @@ func worker(jobs <-chan string, wg *sync.WaitGroup, cfg *Config, stats *Stats, t
 		if response.RecordType == "CNAME" {
 			stats.incrementCNAME()
 			recordTypeColor = "[fuchsia]CNAME[-]"
-			ptr = fmt.Sprintf("%s -> %s", ptr, response.Target)
+			ptr = fmt.Sprintf("%s -> %s", strings.ToLower(ptr), strings.ToLower(response.Target))
 		}
 
 		var line string
@@ -520,6 +521,7 @@ func main() {
 	seed := flag.Int64("s", 0, "Seed for IP generation (0 for random)")
 	shard := flag.String("shard", "", "Shard specification (e.g., 1/4 for first shard of 4)")
 	loop := flag.Bool("l", false, "Loop continuously after completion")
+	jsonOutput := flag.Bool("j", false, "Output NDJSON to stdout (no TUI)")
 	flag.Parse()
 
 	shardNum, totalShards, err := parseShardArg(*shard)
@@ -631,8 +633,9 @@ func main() {
 					}
 
 					// First line: stats
-					statsLine := fmt.Sprintf(" [aqua]Elapsed:[:-] [white]%s [gray]│[-] [aqua]Count:[:-] [white]%s [gray]│[-] [aqua]Progress:[:-] [darkgray]%.2f%%[-] [gray]│[-] [aqua]Rate:[:-] %s [gray]│[-] [aqua]CNAMEs:[:-] [yellow]%s[-][darkgray] (%.1f%%)[-] [gray]│[-] [aqua]Successful:[:-] [green]✓ %s[-][darkgray] (%.1f%%)[-] [gray]│[-] [aqua]Failed:[:-] [red]✗ %s[-][darkgray] (%.1f%%)[-]\n",
+					statsLine := fmt.Sprintf(" [aqua]Elapsed:[:-] [white]%s [gray]│[-] [aqua]Round:[:-] [white]%d [gray]│[-] [aqua]Count:[:-] [white]%s [gray]│[-] [aqua]Progress:[:-] [darkgray]%.2f%%[-] [gray]│[-] [aqua]Rate:[:-] %s [gray]│[-] [aqua]CNAMEs:[:-] [yellow]%s[-][darkgray] (%.1f%%)[-] [gray]│[-] [aqua]Successful:[:-] [green]✓ %s[-][darkgray] (%.1f%%)[-] [gray]│[-] [aqua]Failed:[:-] [red]✗ %s[-][darkgray] (%.1f%%)[-]\n",
 						formatDuration(time.Since(stats.startTime)),
+						atomic.LoadUint64(&stats.round)+1,
 						formatNumber(processed),
 						percent,
 						colorizeSpeed(avgSpeed),
@@ -670,6 +673,142 @@ func main() {
 		}
 	}()
 
+	if *jsonOutput {
+		// JSON-only mode
+		jobs := make(chan string, cfg.concurrency)
+		var wg sync.WaitGroup
+
+		// Start workers
+		for i := 0; i < cfg.concurrency; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for ip := range jobs {
+					var response DNSResponse
+					var err error
+					var server string
+
+					if len(cfg.dnsServers) > 0 {
+						response, server, err = lookupWithRetry(ip, cfg)
+						if idx := strings.Index(server, ":"); idx != -1 {
+							server = server[:idx]
+						}
+					} else {
+						names, err := net.LookupAddr(ip)
+						if err == nil {
+							response = DNSResponse{Names: names, RecordType: "PTR"}
+						}
+					}
+
+					if err != nil {
+						if cfg.debug {
+							errRecord := formatErrorAsHostname(err)
+							record := struct {
+								Seen       string `json:"seen"`
+								IP         string `json:"ip"`
+								Nameserver string `json:"nameserver"`
+								Record     string `json:"record"`
+								RecordType string `json:"record_type"`
+								TTL        uint32 `json:"ttl"`
+							}{
+								Seen:       time.Now().Format(time.RFC3339),
+								IP:         ip,
+								Nameserver: server,
+								Record:     errRecord,
+								RecordType: "ERR",
+								TTL:        0,
+							}
+							if data, err := json.Marshal(record); err == nil {
+								fmt.Println(string(data))
+							}
+						}
+						continue
+					}
+
+					if len(response.Names) == 0 {
+						if cfg.debug {
+							record := struct {
+								Seen       string `json:"seen"`
+								IP         string `json:"ip"`
+								Nameserver string `json:"nameserver"`
+								Record     string `json:"record"`
+								RecordType string `json:"record_type"`
+								TTL        uint32 `json:"ttl"`
+							}{
+								Seen:       time.Now().Format(time.RFC3339),
+								IP:         ip,
+								Nameserver: server,
+								Record:     "FAIL.NO-PTR-RECORD.in-addr.arpa",
+								RecordType: "ERR",
+								TTL:        0,
+							}
+							if data, err := json.Marshal(record); err == nil {
+								fmt.Println(string(data))
+							}
+						}
+						continue
+					}
+
+					ptr := ""
+					for _, name := range response.Names {
+						if cleaned := strings.TrimSpace(strings.TrimSuffix(name, ".")); cleaned != "" {
+							ptr = cleaned
+							break
+						}
+					}
+
+					if ptr == "" {
+						continue
+					}
+
+					record := struct {
+						Seen       string `json:"seen"`
+						IP         string `json:"ip"`
+						Nameserver string `json:"nameserver"`
+						Record     string `json:"record"`
+						RecordType string `json:"record_type"`
+						TTL        uint32 `json:"ttl"`
+					}{
+						Seen:       time.Now().Format(time.RFC3339),
+						IP:         ip,
+						Nameserver: server,
+						Record:     response.Target,
+						RecordType: response.RecordType,
+						TTL:        response.TTL,
+					}
+
+					if response.RecordType != "CNAME" {
+						record.Record = ptr
+					}
+
+					if data, err := json.Marshal(record); err == nil {
+						fmt.Println(string(data))
+					}
+				}
+			}()
+		}
+
+		// Feed IPs to workers
+		for {
+			stream, err := golcg.IPStream("0.0.0.0/0", shardNum, totalShards, int(*seed), nil)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Error creating IP stream: %v\n", err)
+				return
+			}
+
+			for ip := range stream {
+				jobs <- ip
+			}
+
+			if !cfg.loop {
+				break
+			}
+		}
+		close(jobs)
+		wg.Wait()
+		return
+	}
+
 	jobs := make(chan string, cfg.concurrency)
 
 	go func() {
@@ -746,23 +885,26 @@ func writeNDJSON(cfg *Config, timestamp time.Time, ip, server, ptr, recordType,
 	}
 
 	record := struct {
-		Timestamp  string `json:"timestamp"`
-		IPAddr     string `json:"ip_addr"`
-		DNSServer  string `json:"dns_server"`
-		PTRRecord  string `json:"ptr_record"`
+		Seen       string `json:"seen"`
+		IP         string `json:"ip"`
+		Nameserver string `json:"nameserver"`
+		Record     string `json:"record"`
 		RecordType string `json:"record_type"`
-		Target     string `json:"target,omitempty"`
 		TTL        uint32 `json:"ttl"`
 	}{
-		Timestamp:  timestamp.Format(time.RFC3339),
-		IPAddr:     ip,
-		DNSServer:  server,
-		PTRRecord:  ptr,
+		Seen:       timestamp.Format(time.RFC3339),
+		IP:         ip,
+		Nameserver: server,
+		Record:     target, // For CNAME records, use the target
 		RecordType: recordType,
-		Target:     target,
 		TTL:        ttl,
 	}
 
+	// If it's not a CNAME, use the PTR record
+	if recordType != "CNAME" {
+		record.Record = ptr
+	}
+
 	if data, err := json.Marshal(record); err == nil {
 		cfg.mu.Lock()
 		cfg.outputFile.Write(data)
@@ -828,3 +970,37 @@ func colorizeTTL(ttl uint32) string {
 		return fmt.Sprintf("[gray]%-6d[-]", ttl)
 	}
 }
+
+func formatErrorAsHostname(err error) string {
+	errMsg := err.Error()
+	if idx := strings.LastIndex(errMsg, ": "); idx != -1 {
+		errMsg = errMsg[idx+2:]
+	}
+
+	switch {
+	case strings.Contains(errMsg, "i/o timeout"):
+		return "FAIL.TIMEOUT.in-addr.arpa"
+	case strings.Contains(errMsg, "Server Failure"):
+		return "FAIL.SERVER-FAILURE.in-addr.arpa"
+	case strings.Contains(errMsg, "No Such Domain"):
+		return "FAIL.NON-AUTHORITATIVE.in-addr.arpa"
+	case strings.Contains(errMsg, "refused"):
+		return "FAIL.REFUSED.in-addr.arpa"
+	case strings.Contains(errMsg, "no such host"):
+		return "FAIL.NO-SUCH-HOST.in-addr.arpa"
+	case strings.Contains(errMsg, "connection refused"):
+		return "FAIL.CONNECTION-REFUSED.in-addr.arpa"
+	case strings.Contains(errMsg, "network is unreachable"):
+		return "FAIL.NETWORK-UNREACHABLE.in-addr.arpa"
+	case strings.Contains(errMsg, "no route to host"):
+		return "FAIL.NO-ROUTE.in-addr.arpa"
+	case strings.Contains(errMsg, "Format error"):
+		return "FAIL.FORMAT-ERROR.in-addr.arpa"
+	case strings.Contains(errMsg, "Not Implemented"):
+		return "FAIL.NOT-IMPLEMENTED.in-addr.arpa"
+	case strings.Contains(errMsg, "truncated"):
+		return "FAIL.TRUNCATED.in-addr.arpa"
+	default:
+		return fmt.Sprintf("FAIL.%s.in-addr.arpa", strings.ReplaceAll(strings.ToUpper(errMsg), " ", "-"))
+	}
+}