added -j -json arg for cli json output for live pipeline ingestion usage

This commit is contained in:
Dionysus 2025-01-05 07:05:00 -05:00
parent ce0d30666b
commit c78e1dd1b3
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
2 changed files with 122 additions and 10 deletions

17
paused.conf Normal file
View File

@ -0,0 +1,17 @@
# resume information
resume-index = 604
seed = 10352559912596683105
rate = 100
shard = 1/1
nocapture = servername
output-filename = /dev/stdout
output-format = ndjson
adapter-ip = 172.16.0.3
# TARGET SELECTION (IP, PORTS, EXCLUDES)
ports = 80,443
range = 0.0.0.0-1.1.1.0
range = 1.1.1.2-255.255.255.255

View File

@ -520,6 +520,7 @@ func main() {
seed := flag.Int64("s", 0, "Seed for IP generation (0 for random)") seed := flag.Int64("s", 0, "Seed for IP generation (0 for random)")
shard := flag.String("shard", "", "Shard specification (e.g., 1/4 for first shard of 4)") shard := flag.String("shard", "", "Shard specification (e.g., 1/4 for first shard of 4)")
loop := flag.Bool("l", false, "Loop continuously after completion") loop := flag.Bool("l", false, "Loop continuously after completion")
jsonOutput := flag.Bool("j", false, "Output NDJSON to stdout (no TUI)")
flag.Parse() flag.Parse()
shardNum, totalShards, err := parseShardArg(*shard) shardNum, totalShards, err := parseShardArg(*shard)
@ -670,6 +671,97 @@ func main() {
} }
}() }()
if *jsonOutput {
// JSON-only mode
jobs := make(chan string, cfg.concurrency)
var wg sync.WaitGroup
// Start workers
for i := 0; i < cfg.concurrency; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for ip := range jobs {
var response DNSResponse
var err error
var server string
if len(cfg.dnsServers) > 0 {
response, server, err = lookupWithRetry(ip, cfg)
if idx := strings.Index(server, ":"); idx != -1 {
server = server[:idx]
}
} else {
names, err := net.LookupAddr(ip)
if err == nil {
response = DNSResponse{Names: names, RecordType: "PTR"}
}
}
if err != nil || len(response.Names) == 0 {
continue
}
ptr := ""
for _, name := range response.Names {
if cleaned := strings.TrimSpace(strings.TrimSuffix(name, ".")); cleaned != "" {
ptr = cleaned
break
}
}
if ptr == "" {
continue
}
record := struct {
Seen string `json:"seen"`
IP string `json:"ip"`
Nameserver string `json:"nameserver"`
Record string `json:"record"`
RecordType string `json:"record_type"`
TTL uint32 `json:"ttl"`
}{
Seen: time.Now().Format(time.RFC3339),
IP: ip,
Nameserver: server,
Record: response.Target,
RecordType: response.RecordType,
TTL: response.TTL,
}
if response.RecordType != "CNAME" {
record.Record = ptr
}
if data, err := json.Marshal(record); err == nil {
fmt.Println(string(data))
}
}
}()
}
// Feed IPs to workers
for {
stream, err := golcg.IPStream("0.0.0.0/0", shardNum, totalShards, int(*seed), nil)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating IP stream: %v\n", err)
return
}
for ip := range stream {
jobs <- ip
}
if !cfg.loop {
break
}
}
close(jobs)
wg.Wait()
return
}
jobs := make(chan string, cfg.concurrency) jobs := make(chan string, cfg.concurrency)
go func() { go func() {
@ -746,23 +838,26 @@ func writeNDJSON(cfg *Config, timestamp time.Time, ip, server, ptr, recordType,
} }
record := struct { record := struct {
Timestamp string `json:"timestamp"` Seen string `json:"seen"`
IPAddr string `json:"ip_addr"` IP string `json:"ip"`
DNSServer string `json:"dns_server"` Nameserver string `json:"nameserver"`
PTRRecord string `json:"ptr_record"` Record string `json:"record"`
RecordType string `json:"record_type"` RecordType string `json:"record_type"`
Target string `json:"target,omitempty"`
TTL uint32 `json:"ttl"` TTL uint32 `json:"ttl"`
}{ }{
Timestamp: timestamp.Format(time.RFC3339), Seen: timestamp.Format(time.RFC3339),
IPAddr: ip, IP: ip,
DNSServer: server, Nameserver: server,
PTRRecord: ptr, Record: target, // For CNAME records, use the target
RecordType: recordType, RecordType: recordType,
Target: target,
TTL: ttl, TTL: ttl,
} }
// If it's not a CNAME, use the PTR record
if recordType != "CNAME" {
record.Record = ptr
}
if data, err := json.Marshal(record); err == nil { if data, err := json.Marshal(record); err == nil {
cfg.mu.Lock() cfg.mu.Lock()
cfg.outputFile.Write(data) cfg.outputFile.Write(data)