sup roarie

README.md

@@ -16,10 +16,10 @@ A high-performance concurrent web scanner written in Python. HTTPZ efficiently s
 
 ## Installation
 
-### Via pip (recommended)
+### Via pip *(recommended)*
 ```bash
 # Install from PyPI
-pip install httpz-scanner
+pip install httpz_scanner
 
 # The 'httpz' command will now be available in your terminal
 httpz --help
@@ -39,44 +39,82 @@ pip install -r requirements.txt
 
 Basic usage:
 ```bash
-python -m httpz-scanner domains.txt
+python -m httpz_scanner domains.txt
 ```
 
 Scan with all flags enabled and output to JSONL:
 ```bash
-python -m httpz-scanner domains.txt -all -c 100 -o results.jsonl -j -p
+python -m httpz_scanner domains.txt -all -c 100 -o results.jsonl -j -p
 ```
 
 Read from stdin:
 ```bash
-cat domains.txt | python -m httpz-scanner - -all -c 100
+cat domains.txt | python -m httpz_scanner - -all -c 100
-echo "example.com" | python -m httpz-scanner - -all
+echo "example.com" | python -m httpz_scanner - -all
 ```
 
 Filter by status codes and follow redirects:
 ```bash
-python -m httpz-scanner domains.txt -mc 200,301-399 -ec 404,500 -fr -p
+python -m httpz_scanner domains.txt -mc 200,301-399 -ec 404,500 -fr -p
 ```
 
 Show specific fields with custom timeout and resolvers:
 ```bash
-python -m httpz-scanner domains.txt -sc -ti -i -tls -to 10 -r resolvers.txt
+python -m httpz_scanner domains.txt -sc -ti -i -tls -to 10 -r resolvers.txt
 ```
 
 Full scan with all options:
 ```bash
-python -m httpz-scanner domains.txt -c 100 -o output.jsonl -j -all -to 10 -mc 200,301 -ec 404,500 -p -ax -r resolvers.txt
+python -m httpz_scanner domains.txt -c 100 -o output.jsonl -j -all -to 10 -mc 200,301 -ec 404,500 -p -ax -r resolvers.txt
 ```
 
+### Distributed Scanning
+Split scanning across multiple machines using the `--shard` argument:
+
+```bash
+# Machine 1
+httpz domains.txt --shard 1/3
+
+# Machine 2
+httpz domains.txt --shard 2/3
+
+# Machine 3
+httpz domains.txt --shard 3/3
+```
+
+Each machine will process a different subset of domains without overlap. For example, with 3 shards:
+- Machine 1 processes lines 0,3,6,9,...
+- Machine 2 processes lines 1,4,7,10,...
+- Machine 3 processes lines 2,5,8,11,...
+
+This allows efficient distribution of large scans across multiple machines.
+
 ### Python Library
 ```python
 import asyncio
+import urllib.request
 from httpz_scanner import HTTPZScanner
 
-async def scan_domains():
+async def scan_from_list() -> list:
+    with urllib.request.urlopen('https://example.com/domains.txt') as response:
+        content = response.read().decode()
+        return [line.strip() for line in content.splitlines() if line.strip()][:20]
+    
+async def scan_from_url():
+    with urllib.request.urlopen('https://example.com/domains.txt') as response:
+        for line in response:
+            if line := line.strip():
+                yield line.decode().strip()
+
+async def scan_from_file():
+    with open('domains.txt', 'r') as file:
+        for line in file:
+            if line := line.strip():
+                yield line
+
+async def main():
     # Initialize scanner with all possible options (showing defaults)
     scanner = HTTPZScanner(
-        # Core settings
         concurrent_limit=100,   # Number of concurrent requests
         timeout=5,              # Request timeout in seconds
         follow_redirects=False, # Follow redirects (max 10)
@@ -86,6 +124,7 @@ async def scan_domains():
         show_progress=False,    # Show progress counter
         debug_mode=False,       # Show error states and debug info
         jsonl_output=False,     # Output in JSONL format
+        shard=None,             # Tuple of (shard_index, total_shards) for distributed scanning
         
         # Control which fields to show (all False by default unless show_fields is None)
         show_fields={
@@ -103,69 +142,49 @@ async def scan_domains():
         },
         
         # Filter results
-        match_codes={200, 301, 302},    # Only show these status codes
+        match_codes={200,301,302},  # Only show these status codes
-        exclude_codes={404, 500, 503}   # Exclude these status codes
+        exclude_codes={404,500,503} # Exclude these status codes
     )
 
-    # Initialize resolvers (required before scanning)
+    # Example 1: Process file
-    await scanner.init()
+    print('\nProcessing file:')
+    async for result in scanner.scan(scan_from_file()):
+        print(f"{result['domain']}: {result['status']}")
 
-    # Scan domains from file
+    # Example 2: Stream URLs
-    await scanner.scan('domains.txt')
+    print('\nStreaming URLs:')
+    async for result in scanner.scan(scan_from_url()):
+        print(f"{result['domain']}: {result['status']}")
 
-    # Or scan from stdin
+    # Example 3: Process list
-    await scanner.scan('-')
+    print('\nProcessing list:')
+    domains = await scan_from_list()
+    async for result in scanner.scan(domains):
+        print(f"{result['domain']}: {result['status']}")
 
 if __name__ == '__main__':
-    asyncio.run(scan_domains())
+    asyncio.run(main())
 ```
 
-The scanner will return results in this format:
+The scanner accepts various input types:
-```python
+- File paths (string)
-{
+- Lists/tuples of domains
-    'domain': 'example.com',           # Base domain
+- stdin (using '-')
-    'url': 'https://example.com',      # Full URL
+- Async generators that yield domains
-    'status': 200,                     # HTTP status code
+
-    'port': 443,                       # Port number
+All inputs support sharding for distributed scanning using the `shard` parameter.
-    'title': 'Example Domain',         # Page title
-    'body': 'Example body text...',    # Body preview
-    'content_type': 'text/html',       # Content type
-    'content_length': '12345',         # Content length
-    'ips': ['93.184.216.34'],         # IP addresses
-    'cname': 'cdn.example.com',        # CNAME record
-    'nameservers': ['ns1.example.com'],# Nameservers
-    'favicon_hash': '123456789',       # Favicon hash
-    'headers': {                       # Response headers
-        'Server': 'nginx',
-        'Content-Type': 'text/html'
-    },
-    'redirect_chain': [               # Redirect history
-        'http://example.com',
-        'https://example.com'
-    ],
-    'tls': {                         # TLS certificate info
-        'fingerprint': 'sha256...',
-        'common_name': 'example.com',
-        'issuer': 'Let\'s Encrypt',
-        'alt_names': ['www.example.com'],
-        'not_before': '2023-01-01T00:00:00',
-        'not_after': '2024-01-01T00:00:00',
-        'version': 3,
-        'serial_number': 'abcdef1234'
-    }
-}
-```
 
 ## Arguments
 
 | Argument      | Long Form        | Description                                                 |
-|-----------|------------------|-------------------------------------------------------------|
+|---------------|------------------|-------------------------------------------------------------|
-| `file`    | -                | File containing domains *(one per line)*, use `-` for stdin |
+| `file`        |                  | File containing domains *(one per line)*, use `-` for stdin |
 | `-d`          | `--debug`        | Show error states and debug information                     |
 | `-c N`        | `--concurrent N` | Number of concurrent checks *(default: 100)*                |
 | `-o FILE`     | `--output FILE`  | Output file path *(JSONL format)*                           |
 | `-j`          | `--jsonl`        | Output JSON Lines format to console                         |
 | `-all`        | `--all-flags`    | Enable all output flags                                     |
+| `-sh`         | `--shard N/T`    | Process shard N of T total shards *(e.g., 1/3)*             |
 
 ### Output Field Flags
 

httpz-scanner/cli.py

@@ -1,174 +0,0 @@
-#!/usr/bin/env python3
-# HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/cli.py
-
-import argparse
-import asyncio
-import logging
-import os
-import sys
-
-from .colors  import Colors
-from .scanner import HTTPZScanner
-from .utils   import SILENT_MODE, info
-
-def setup_logging(level='INFO', log_to_disk=False):
-    '''
-    Setup logging configuration
-    
-    :param level: Logging level (INFO or DEBUG)
-    :param log_to_disk: Whether to also log to file
-    '''
-    class ColoredFormatter(logging.Formatter):
-        def formatTime(self, record, datefmt=None):
-            # Format: MM-DD HH:MM
-            from datetime import datetime
-            dt = datetime.fromtimestamp(record.created)
-            return f"{Colors.GRAY}{dt.strftime('%m-%d %H:%M')}{Colors.RESET}"
-        
-        def format(self, record):
-            return f'{self.formatTime(record)} {record.getMessage()}'
-    
-    handlers = []
-    
-    # Console handler
-    console = logging.StreamHandler()
-    console.setFormatter(ColoredFormatter())
-    handlers.append(console)
-    
-    # File handler
-    if log_to_disk:
-        os.makedirs('logs', exist_ok=True)
-        file_handler = logging.FileHandler(f'logs/httpz.log')
-        file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
-        handlers.append(file_handler)
-    
-    # Setup logger
-    logging.basicConfig(
-        level=getattr(logging, level.upper()),
-        handlers=handlers
-    )
-
-def parse_status_codes(codes_str: str) -> set:
-    '''
-    Parse comma-separated status codes and ranges into a set of integers
-    
-    :param codes_str: Comma-separated status codes (e.g., "200,301-399,404,500-503")
-    '''
-    
-    codes = set()
-    try:
-        for part in codes_str.split(','):
-            if '-' in part:
-                start, end = map(int, part.split('-'))
-                codes.update(range(start, end + 1))
-            else:
-                codes.add(int(part))
-        return codes
-    except ValueError:
-        raise argparse.ArgumentTypeError('Invalid status code format. Use comma-separated numbers or ranges (e.g., 200,301-399,404,500-503)')
-
-async def main():
-    parser = argparse.ArgumentParser(
-        description=f'{Colors.GREEN}Hyper-fast HTTP Scraping Tool{Colors.RESET}',
-        formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-
-    # Add arguments
-    parser.add_argument('file', nargs='?', default='-', help='File containing domains to check (one per line), use - for stdin')
-    parser.add_argument('-all', '--all-flags', action='store_true', help='Enable all output flags')
-    parser.add_argument('-d', '--debug', action='store_true', help='Show error states and debug information')
-    parser.add_argument('-c', '--concurrent', type=int, default=100, help='Number of concurrent checks')
-    parser.add_argument('-j', '--jsonl', action='store_true', help='Output JSON Lines format to console')
-    parser.add_argument('-o', '--output', help='Output file path (JSONL format)')
-    
-    # Output field flags
-    parser.add_argument('-b', '--body', action='store_true', help='Show body preview')
-    parser.add_argument('-cn', '--cname', action='store_true', help='Show CNAME records')
-    parser.add_argument('-cl', '--content-length', action='store_true', help='Show content length')
-    parser.add_argument('-ct', '--content-type', action='store_true', help='Show content type')
-    parser.add_argument('-f', '--favicon', action='store_true', help='Show favicon hash')
-    parser.add_argument('-fr', '--follow-redirects', action='store_true', help='Follow redirects (max 10)')
-    parser.add_argument('-hr', '--headers', action='store_true', help='Show response headers')
-    parser.add_argument('-i', '--ip', action='store_true', help='Show IP addresses')
-    parser.add_argument('-sc', '--status-code', action='store_true', help='Show status code')
-    parser.add_argument('-ti', '--title', action='store_true', help='Show page title')
-    parser.add_argument('-tls', '--tls-info', action='store_true', help='Show TLS certificate information')
-    
-    # Other arguments
-    parser.add_argument('-ax', '--axfr', action='store_true', help='Try AXFR transfer against nameservers')
-    parser.add_argument('-ec', '--exclude-codes', type=parse_status_codes, help='Exclude these status codes (comma-separated, e.g., 404,500)')
-    parser.add_argument('-mc', '--match-codes', type=parse_status_codes, help='Only show these status codes (comma-separated, e.g., 200,301,404)')
-    parser.add_argument('-p', '--progress', action='store_true', help='Show progress counter')
-    parser.add_argument('-r', '--resolvers', help='File containing DNS resolvers (one per line)')
-    parser.add_argument('-to', '--timeout', type=int, default=5, help='Request timeout in seconds')
-    
-    args = parser.parse_args()
-
-    # Setup logging based on arguments
-    global SILENT_MODE
-    SILENT_MODE = args.jsonl
-
-    if not SILENT_MODE:
-        if args.debug:
-            setup_logging(level='DEBUG', log_to_disk=True)
-        else:
-            setup_logging(level='INFO')
-
-        if args.file == '-':
-            info('Reading domains from stdin')
-        else:
-            info(f'Processing file: {args.file}')
-
-    # Setup show_fields
-    show_fields = {
-        'status_code'      : args.all_flags or args.status_code,
-        'content_type'     : args.all_flags or args.content_type,
-        'content_length'   : args.all_flags or args.content_length,
-        'title'            : args.all_flags or args.title,
-        'body'             : args.all_flags or args.body,
-        'ip'               : args.all_flags or args.ip,
-        'favicon'          : args.all_flags or args.favicon,
-        'headers'          : args.all_flags or args.headers,
-        'follow_redirects' : args.all_flags or args.follow_redirects,
-        'cname'            : args.all_flags or args.cname,
-        'tls'              : args.all_flags or args.tls_info
-    }
-
-    # If no fields specified show all
-    if not any(show_fields.values()):
-        show_fields = {k: True for k in show_fields}
-
-    try:
-        # Create scanner instance
-        scanner = HTTPZScanner(
-            concurrent_limit=args.concurrent,
-            timeout=args.timeout,
-            follow_redirects=args.all_flags or args.follow_redirects,
-            check_axfr=args.axfr,
-            resolver_file=args.resolvers,
-            output_file=args.output,
-            show_progress=args.progress,
-            debug_mode=args.debug,
-            jsonl_output=args.jsonl,
-            show_fields=show_fields,
-            match_codes=args.match_codes,
-            exclude_codes=args.exclude_codes
-        )
-
-        # Run the scanner with file/stdin input
-        await scanner.scan(args.file)
-
-    except KeyboardInterrupt:
-        logging.warning('Process interrupted by user')
-        sys.exit(1)
-    except Exception as e:
-        logging.error(f'Unexpected error: {str(e)}')
-        sys.exit(1)
-
-def run():
-    '''Entry point for the CLI'''
-    asyncio.run(main())
-
-if __name__ == '__main__':
-    run() 

httpz-scanner/scanner.py

@@ -1,239 +0,0 @@
-#!/usr/bin/env python3
-# HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/scanner.py
-
-import asyncio
-import json
-import random
-import sys
-
-try:
-    import aiohttp
-except ImportError:
-    raise ImportError('missing aiohttp module (pip install aiohttp)')
-
-try:
-    import bs4
-except ImportError:
-    raise ImportError('missing bs4 module (pip install beautifulsoup4)')
-
-from .dns        import resolve_all_dns, load_resolvers
-from .formatters import format_console_output
-from .colors     import Colors
-from .parsers    import parse_domain_url, get_cert_info, get_favicon_hash
-from .utils      import debug, info, USER_AGENTS, input_generator
-
-
-class HTTPZScanner:
-    '''Core scanner class for HTTP domain checking'''
-    
-    def __init__(self, concurrent_limit = 100, timeout = 5, follow_redirects = False, check_axfr = False, resolver_file = None, output_file = None, show_progress = False, debug_mode = False, jsonl_output = False, show_fields = None, match_codes = None, exclude_codes = None):
-        '''
-        Initialize the HTTPZScanner class
-        
-        :param concurrent_limit: Maximum number of concurrent requests
-        :param timeout: Request timeout in seconds
-        :param follow_redirects: Follow redirects
-        :param check_axfr: Check for AXFR
-        :param resolver_file: Path to resolver file
-        :param output_file: Path to output file
-        :param show_progress: Show progress bar
-        :param debug_mode: Enable debug mode
-        :param jsonl_output: Output in JSONL format
-        :param show_fields: Fields to show
-        :param match_codes: Status codes to match
-        :param exclude_codes: Status codes to exclude
-        '''
-
-        self.concurrent_limit = concurrent_limit
-        self.timeout          = timeout
-        self.follow_redirects = follow_redirects
-        self.check_axfr       = check_axfr
-        self.resolver_file    = resolver_file
-        self.output_file      = output_file
-        self.show_progress    = show_progress
-        self.debug_mode       = debug_mode
-        self.jsonl_output     = jsonl_output
-
-        self.show_fields = show_fields or {
-            'status_code'      : True,
-            'content_type'     : True,
-            'content_length'   : True,
-            'title'            : True,
-            'body'             : True,
-            'ip'               : True,
-            'favicon'          : True,
-            'headers'          : True,
-            'follow_redirects' : True,
-            'cname'            : True,
-            'tls'              : True
-        }
-
-        self.match_codes       = match_codes
-        self.exclude_codes     = exclude_codes
-        self.resolvers         = None
-        self.processed_domains = 0
-
-
-    async def init(self):
-        '''Initialize resolvers - must be called before scanning'''
-        self.resolvers = await load_resolvers(self.resolver_file)
-
-
-    async def check_domain(self, session: aiohttp.ClientSession, domain: str):
-        '''Check a single domain and return results'''
-        nameserver = random.choice(self.resolvers) if self.resolvers else None
-        base_domain, port, protocols = parse_domain_url(domain)
-        
-        result = {
-            'domain'  : base_domain,
-            'status'  : 0,
-            'url'     : protocols[0],
-            'port'    : port,
-        }
-
-        # Try each protocol
-        for url in protocols:
-            try:
-                # Set random user agent for each request
-                headers = {'User-Agent': random.choice(USER_AGENTS)}
-                
-                async with session.get(url, timeout=self.timeout, 
-                                     allow_redirects=self.follow_redirects,
-                                     max_redirects=10 if self.follow_redirects else 0,
-                                     headers=headers) as response:
-                    
-                    result['status'] = response.status
-                    
-                    # Early exit if status code doesn't match criteria
-                    if self.match_codes and result['status'] not in self.match_codes:
-                        return result
-                    if self.exclude_codes and result['status'] in self.exclude_codes:
-                        return result
-
-                    # Continue with full processing only if status code matches criteria
-                    result['url'] = str(response.url)
-                    
-                    # Add headers if requested
-                    headers = dict(response.headers)
-                    if headers and (self.show_fields.get('headers') or self.show_fields.get('all_flags')):
-                        result['headers'] = headers
-                    else:
-                        # Only add content type/length if headers aren't included
-                        if content_type := response.headers.get('content-type', '').split(';')[0]:
-                            result['content_type'] = content_type
-                        if content_length := response.headers.get('content-length'):
-                            result['content_length'] = content_length
-                    
-                    # Only add redirect chain if it exists
-                    if self.follow_redirects and response.history:
-                        result['redirect_chain'] = [str(h.url) for h in response.history] + [str(response.url)]
-
-                    # Do DNS lookups only if we're going to use the result
-                    ips, cname, nameservers, _ = await resolve_all_dns(
-                        base_domain, self.timeout, nameserver, self.check_axfr
-                    )
-                    
-                    # Only add DNS fields if they have values
-                    if ips:
-                        result['ips'] = ips
-                    if cname:
-                        result['cname'] = cname
-                    if nameservers:
-                        result['nameservers'] = nameservers
-
-                    # Only add TLS info if available
-                    if response.url.scheme == 'https':
-                        try:
-                            if ssl_object := response._protocol.transport.get_extra_info('ssl_object'):
-                                if tls_info := await get_cert_info(ssl_object, str(response.url)):
-                                    # Only add TLS fields that have values
-                                    result['tls'] = {k: v for k, v in tls_info.items() if v}
-                        except AttributeError:
-                            debug(f'Failed to get SSL info for {url}')
-
-                    html = (await response.text())[:1024*1024]
-                    soup = bs4.BeautifulSoup(html, 'html.parser')
-                    
-                    # Only add title if it exists
-                    if soup.title and soup.title.string:
-                        result['title'] = ' '.join(soup.title.string.strip().split()).rstrip('.')[:300]
-                    
-                    # Only add body if it exists
-                    if body_text := soup.get_text():
-                        result['body'] = ' '.join(body_text.split()).rstrip('.')[:500]
-                    
-                    # Only add favicon hash if it exists
-                    if favicon_hash := await get_favicon_hash(session, url, html):
-                        result['favicon_hash'] = favicon_hash
-                    
-                    break
-            except Exception as e:
-                debug(f'Error checking {url}: {str(e)}')
-                result['status'] = -1
-                continue
-
-        return result
-
-
-    async def process_result(self, result):
-        '''
-        Process and output a single result
-        
-        :param result: result to process
-        '''
-
-        formatted = format_console_output(result, self.debug_mode, self.show_fields, self.match_codes, self.exclude_codes)
-        
-        if formatted:
-            # Write to file if specified
-            if self.output_file:
-                if (not self.match_codes or result['status'] in self.match_codes) and \
-                   (not self.exclude_codes or result['status'] not in self.exclude_codes):
-                    async with aiohttp.ClientSession() as session:
-                        with open(self.output_file, 'a') as f:
-                            json.dump(result, f, ensure_ascii=False)
-                            f.write('\n')
-            
-            # Console output
-            if self.jsonl_output:
-                print(json.dumps(result))
-            else:
-                self.processed_domains += 1
-                if self.show_progress:
-                    info(f"{Colors.GRAY}[{self.processed_domains:,}]{Colors.RESET} {formatted}")
-                else:
-                    info(formatted)
-
-
-    async def scan(self, input_source):
-        '''
-        Scan domains from a file or stdin
-        
-        :param input_source: Path to file or '-' for stdin
-        '''
-        if not self.resolvers:
-            await self.init()
-
-        async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
-            tasks = set()
-            
-            # Process domains with concurrent limit
-            for domain in input_generator(input_source):
-                if len(tasks) >= self.concurrent_limit:
-                    done, tasks = await asyncio.wait(
-                        tasks, return_when=asyncio.FIRST_COMPLETED
-                    )
-                    for task in done:
-                        result = await task
-                        await self.process_result(result)
-
-                task = asyncio.create_task(self.check_domain(session, domain))
-                tasks.add(task)
-
-            # Process remaining tasks
-            if tasks:
-                done, _ = await asyncio.wait(tasks)
-                for task in done:
-                    result = await task
-                    await self.process_result(result) 

httpz-scanner/utils.py

@@ -1,115 +0,0 @@
-#!/usr/bin/env python3
-# HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/utils.py
-
-import logging
-import sys
-
-
-# Global for silent mode
-SILENT_MODE = False
-
-# List of user agents to randomize requests
-USER_AGENTS = [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.6.5 Chrome/124.0.6367.243 Electron/30.1.2 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 OPR/116.0.0.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:134.0) Gecko/20100101 Firefox/134.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.8.3 Chrome/130.0.6723.191 Electron/33.3.2 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.137 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.5.12 Chrome/120.0.6099.283 Electron/28.2.3 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
-    "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 OPR/114.0.0.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.7.7 Chrome/128.0.6613.186 Electron/32.2.5 Safari/537.36"
-]
-
-
-def debug(msg: str): 
-    if not SILENT_MODE: logging.debug(msg)
-def error(msg: str):
-    if not SILENT_MODE: logging.error(msg)
-def info(msg: str):
-    if not SILENT_MODE: logging.info(msg)
-def warning(msg: str):
-    if not SILENT_MODE: logging.warning(msg)
-
-
-def human_size(size_bytes: int) -> str:
-    '''
-    Convert bytes to human readable string
-    
-    :param size_bytes: size in bytes
-    '''
-
-    if not size_bytes:
-        return '0B'
-    
-    units      = ('B', 'KB', 'MB', 'GB')
-    size       = float(size_bytes)
-    unit_index = 0
-    
-    while size >= 1024 and unit_index < len(units) - 1:
-        size /= 1024
-        unit_index += 1
-    
-    return f'{size:.1f}{units[unit_index]}'
-
-
-def input_generator(input_source: str):
-    '''
-    Generator function to yield domains from file or stdin
-    
-    :param input_source: file or stdin
-    '''
-    
-    if input_source == '-' or input_source is None:
-        for line in sys.stdin:
-            if line.strip():
-                yield line.strip()
-    else:
-        with open(input_source, 'r') as f:
-            for line in f:
-                if line.strip():
-                    yield line.strip() 

httpz_scanner/__init__.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 # HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/__init__.py
+# httpz_scanner/__init__.py
 
-from .scanner import HTTPZScanner
 from .colors  import Colors
+from .scanner import HTTPZScanner
 
 
-__version__ = '1.0.3' 
+__version__ = '2.1.8'

httpz_scanner/__main__.py

@@ -1,11 +1,14 @@
 #!/usr/bin/env python3
 # HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/__main__.py
+# httpz_scanner/__main__.py
 
 import asyncio
 import sys
+
 from .cli import main
 
+
+
 if __name__ == '__main__':
     try:
         asyncio.run(main())

httpz_scanner/cli.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
+# httpz_scanner/cli.py
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import sys
+
+from datetime import datetime
+
+from .colors     import Colors
+from .formatters import format_console_output
+from .parsers    import parse_status_codes, parse_shard
+from .scanner    import HTTPZScanner
+from .utils      import SILENT_MODE, info
+
+
+def setup_logging(level='INFO', log_to_disk=False):
+    '''
+    Setup logging configuration
+    
+    :param level: Logging level (INFO or DEBUG)
+    :param log_to_disk: Whether to also log to file
+    '''
+
+    class ColoredFormatter(logging.Formatter):
+        def formatTime(self, record):
+            dt = datetime.fromtimestamp(record.created)
+            return f'{Colors.GRAY}{dt.strftime("%m-%d %H:%M")}{Colors.RESET}'
+        
+        def format(self, record):
+            return f'{self.formatTime(record)} {record.getMessage()}'
+    
+    # Setup logging handlers
+    handlers = []
+    
+    # Console handler
+    console = logging.StreamHandler()
+    console.setFormatter(ColoredFormatter())
+    handlers.append(console)
+    
+    # File handler
+    if log_to_disk:
+        os.makedirs('logs', exist_ok=True)
+        file_handler = logging.FileHandler(f'logs/httpz.log')
+        file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+        handlers.append(file_handler)
+    
+    # Setup logger
+    logging.basicConfig(level=getattr(logging, level.upper()), handlers=handlers)
+
+
+async def main():
+    parser = argparse.ArgumentParser(description=f'{Colors.GREEN}Hyper-fast HTTP Scraping Tool{Colors.RESET}', formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    # Add arguments
+    parser.add_argument('file', nargs='?', default='-', help='File containing domains to check (one per line), use - for stdin')
+    parser.add_argument('-all', '--all-flags', action='store_true', help='Enable all output flags')
+    parser.add_argument('-d',   '--debug', action='store_true', help='Show error states and debug information')
+    parser.add_argument('-c',   '--concurrent', type=int, default=100, help='Number of concurrent checks')
+    parser.add_argument('-j',   '--jsonl', action='store_true', help='Output JSON Lines format to console')
+    parser.add_argument('-o',   '--output', help='Output file path (JSONL format)')
+    
+    # Output field flags
+    parser.add_argument('-b',   '--body', action='store_true', help='Show body preview')
+    parser.add_argument('-cn',  '--cname', action='store_true', help='Show CNAME records')
+    parser.add_argument('-cl',  '--content-length', action='store_true', help='Show content length')
+    parser.add_argument('-ct',  '--content-type', action='store_true', help='Show content type')
+    parser.add_argument('-f',   '--favicon', action='store_true', help='Show favicon hash')
+    parser.add_argument('-fr',  '--follow-redirects', action='store_true', help='Follow redirects (max 10)')
+    parser.add_argument('-hr',  '--show-headers', action='store_true', help='Show response headers')
+    parser.add_argument('-i',   '--ip', action='store_true', help='Show IP addresses')
+    parser.add_argument('-sc',  '--status-code', action='store_true', help='Show status code')
+    parser.add_argument('-ti',  '--title', action='store_true', help='Show page title')
+    parser.add_argument('-tls', '--tls-info', action='store_true', help='Show TLS certificate information')
+    
+    # Other arguments
+    parser.add_argument('-ax', '--axfr', action='store_true', help='Try AXFR transfer against nameservers')
+    parser.add_argument('-ec', '--exclude-codes', type=parse_status_codes, help='Exclude these status codes (comma-separated, e.g., 404,500)')
+    parser.add_argument('-mc', '--match-codes', type=parse_status_codes, help='Only show these status codes (comma-separated, e.g., 200,301,404)')
+    parser.add_argument('-p',  '--progress', action='store_true', help='Show progress counter')
+    parser.add_argument('-pd', '--post-data', help='Send POST request with this data')
+    parser.add_argument('-r',  '--resolvers', help='File containing DNS resolvers (one per line)')
+    parser.add_argument('-to', '--timeout', type=int, default=5, help='Request timeout in seconds')
+    
+    # Add shard argument
+    parser.add_argument('-sh','--shard', type=parse_shard, help='Shard index and total shards (e.g., 1/3)')
+
+    # Add this to the argument parser section
+    parser.add_argument('-pa', '--paths', help='Additional paths to check (comma-separated, e.g., ".git/config,.env")')
+    
+    # Add these arguments in the parser section
+    parser.add_argument('-hd', '--headers', help='Custom headers to send with each request (format: "Header1: value1,Header2: value2")')
+    
+    # If no arguments provided, print help and exit
+    if len(sys.argv) == 1:
+        parser.print_help()
+        sys.exit(0)
+    
+    args = parser.parse_args()
+
+    # Setup logging based on arguments
+    global SILENT_MODE
+    SILENT_MODE = args.jsonl
+
+    if not SILENT_MODE:
+        if args.debug:
+            setup_logging(level='DEBUG', log_to_disk=True)
+        else:
+            setup_logging(level='INFO')
+
+        if args.file == '-':
+            info('Reading domains from stdin')
+        else:
+            info(f'Processing file: {args.file}')
+
+    # Setup show_fields
+    show_fields = {
+        'status_code'      : args.all_flags or args.status_code,
+        'content_type'     : args.all_flags or args.content_type,
+        'content_length'   : args.all_flags or args.content_length,
+        'title'            : args.all_flags or args.title,
+        'body'             : args.all_flags or args.body,
+        'ip'               : args.all_flags or args.ip,
+        'favicon'          : args.all_flags or args.favicon,
+        'headers'          : args.all_flags or args.show_headers,
+        'follow_redirects' : args.all_flags or args.follow_redirects,
+        'cname'            : args.all_flags or args.cname,
+        'tls'              : args.all_flags or args.tls_info
+    }
+
+    # If no fields specified show all
+    if not any(show_fields.values()):
+        show_fields = {k: True for k in show_fields}
+
+    try:
+        scanner = HTTPZScanner(
+            concurrent_limit=args.concurrent,
+            timeout=args.timeout,
+            follow_redirects=args.all_flags or args.follow_redirects,
+            check_axfr=args.axfr,
+            resolver_file=args.resolvers,
+            output_file=args.output,
+            show_progress=args.progress,
+            debug_mode=args.debug,
+            jsonl_output=args.jsonl,
+            show_fields=show_fields,
+            match_codes=args.match_codes,
+            exclude_codes=args.exclude_codes,
+            shard=args.shard,
+            paths=args.paths.split(',') if args.paths else None,
+            custom_headers=dict(h.split(': ', 1) for h in args.headers.split(',')) if args.headers else None,
+            post_data=args.post_data
+        )
+
+        count = 0
+        async for result in scanner.scan(args.file):
+            # Write to output file if specified
+            if args.output:
+                with open(args.output, 'a') as f:
+                    f.write(json.dumps(result) + '\n')
+                    f.flush()  # Ensure file output is immediate
+            
+            # Handle JSON output separately
+            if args.jsonl:
+                print(json.dumps(result), flush=True)  # Force flush
+                continue
+
+            # Only output and increment counter if we have content to show for normal output
+            formatted = format_console_output(result, args.debug, show_fields, args.match_codes, args.exclude_codes)
+            if formatted:
+                if args.progress:
+                    count += 1
+                    info(f"[{count}] {formatted}")
+                    sys.stdout.flush()  # Force flush after each domain
+                else:
+                    print(formatted, flush=True)  # Force flush
+
+    except KeyboardInterrupt:
+        logging.warning('Process interrupted by user')
+        sys.exit(1)
+    except Exception as e:
+        logging.error(f'Unexpected error: {str(e)}')
+        sys.exit(1)
+
+
+def run():
+    '''Entry point for the CLI'''
+    asyncio.run(main())
+
+
+
+if __name__ == '__main__':
+    run() 

httpz_scanner/colors.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 # HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/colors.py
+# httpz_scanner/colors.py
 
 class Colors:
     '''ANSI color codes for terminal output'''
-    HEADER     = '\033[95m' # Light purple
+
+    HEADER     = '\033[95m'
     BLUE       = '\033[94m'
     GREEN      = '\033[92m'
     YELLOW     = '\033[93m'
@@ -12,9 +13,9 @@ class Colors:
     BOLD       = '\033[1m'
     UNDERLINE  = '\033[4m'
     RESET      = '\033[0m'
-    PURPLE     = '\033[35m'       # Dark purple
+    PURPLE     = '\033[35m'
-    LIGHT_RED  = '\033[38;5;203m' # Light red
+    LIGHT_RED  = '\033[38;5;203m'
-    DARK_GREEN = '\033[38;5;22m'  # Dark green
+    DARK_GREEN = '\033[38;5;22m'
-    PINK       = '\033[38;5;198m' # Bright pink
+    PINK       = '\033[38;5;198m'
-    GRAY       = '\033[90m'       # Gray color
+    GRAY       = '\033[90m'
-    CYAN       = '\033[96m'       # Cyan color 
+    CYAN       = '\033[96m' 

httpz_scanner/dns.py

@@ -1,17 +1,26 @@
 #!/usr/bin/env python3
 # HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/dns.py
+# httpz_scanner/dns.py
 
 import asyncio
 import os
-import aiohttp
+
-import dns.asyncresolver
+try:
-import dns.query
+    import aiohttp
-import dns.resolver
+except ImportError:
-import dns.zone
+    raise ImportError('missing aiohttp library (pip install aiohttp)')
+
+try:
+    import dns.asyncresolver
+    import dns.query
+    import dns.resolver
+    import dns.zone
+except ImportError:
+    raise ImportError('missing dnspython library (pip install dnspython)')
 
 from .utils import debug, info, SILENT_MODE
 
+
 async def resolve_all_dns(domain: str, timeout: int = 5, nameserver: str = None, check_axfr: bool = False) -> tuple:
     '''
     Resolve all DNS records for a domain
@@ -21,36 +30,35 @@ async def resolve_all_dns(domain: str, timeout: int = 5, nameserver: str = None,
     :param nameserver: Specific nameserver to use
     :param check_axfr: Whether to attempt zone transfer
     '''
+
+    # Setup resolver
     resolver = dns.asyncresolver.Resolver()
     resolver.lifetime = timeout
     if nameserver:
         resolver.nameservers = [nameserver]
     
-    results = await asyncio.gather(*[resolver.resolve(domain, rtype) 
+    # Resolve all DNS records
-                                   for rtype in ('NS', 'A', 'AAAA', 'CNAME')], 
+    results = await asyncio.gather(*[resolver.resolve(domain, rtype) for rtype in ('NS', 'A', 'AAAA', 'CNAME')], return_exceptions=True)
-                                 return_exceptions=True)
     
+    # Parse results
     nameservers = [str(ns).rstrip('.') for ns in results[0]] if isinstance(results[0], dns.resolver.Answer) else []
-    ips = ([str(ip) for ip in results[1]] if isinstance(results[1], dns.resolver.Answer) else []) + \
+    ips         = ([str(ip) for ip in results[1]] if isinstance(results[1], dns.resolver.Answer) else []) + ([str(ip) for ip in results[2]] if isinstance(results[2], dns.resolver.Answer) else [])
-          ([str(ip) for ip in results[2]] if isinstance(results[2], dns.resolver.Answer) else [])
     cname       = str(results[3][0].target).rstrip('.') if isinstance(results[3], dns.resolver.Answer) else None
 
+    # Get NS IPs
     ns_ips = {}
     if nameservers:
-        ns_results = await asyncio.gather(*[resolver.resolve(ns, rtype) 
+        ns_results = await asyncio.gather(*[resolver.resolve(ns, rtype) for ns in nameservers for rtype in ('A', 'AAAA')], return_exceptions=True)
-                                          for ns in nameservers 
-                                          for rtype in ('A', 'AAAA')], 
-                                        return_exceptions=True)
         for i, ns in enumerate(nameservers):
-            ns_ips[ns] = [str(ip) for records in ns_results[i*2:i*2+2] 
+            ns_ips[ns] = [str(ip) for records in ns_results[i*2:i*2+2] if isinstance(records, dns.resolver.Answer) for ip in records]
-                         if isinstance(records, dns.resolver.Answer) 
-                         for ip in records]
 
+    # Attempt zone transfer
     if check_axfr:
         await attempt_axfr(domain, ns_ips, timeout)
 
     return sorted(set(ips)), cname, nameservers, ns_ips
 
+
 async def attempt_axfr(domain: str, ns_ips: dict, timeout: int = 5) -> None:
     '''
     Attempt zone transfer for a domain
@@ -59,28 +67,37 @@ async def attempt_axfr(domain: str, ns_ips: dict, timeout: int = 5) -> None:
     :param ns_ips: Dictionary of nameserver hostnames to their IPs
     :param timeout: Timeout in seconds
     '''
+
     try:
         os.makedirs('axfrout', exist_ok=True)
 
+        # Loop through each NS
         for ns_host, ips in ns_ips.items():
+            # Loop through each NS IP
             for ns_ip in ips:
                 try:
+                    # Attempt zone transfer
                     zone = dns.zone.from_xfr(dns.query.xfr(ns_ip, domain, lifetime=timeout))
+
+                    # Write zone to file
                     with open(f'axfrout/{domain}_{ns_ip}.zone', 'w') as f:
                         zone.to_text(f)
+
                     info(f'[AXFR SUCCESS] {domain} from {ns_host} ({ns_ip})')
                 except Exception as e:
                     debug(f'AXFR failed for {domain} from {ns_ip}: {str(e)}')
     except Exception as e:
         debug(f'Failed AXFR for {domain}: {str(e)}')
 
+
 async def load_resolvers(resolver_file: str = None) -> list:
     '''
     Load DNS resolvers from file or default source
     
     :param resolver_file: Path to file containing resolver IPs
-    :return: List of resolver IPs
     '''
+
+    # Load from file
     if resolver_file:
         try:
             with open(resolver_file) as f:
@@ -90,6 +107,7 @@ async def load_resolvers(resolver_file: str = None) -> list:
         except Exception as e:
             debug(f'Error loading resolvers from {resolver_file}: {str(e)}')
 
+    # Load from GitHub
     async with aiohttp.ClientSession() as session:
         async with session.get('https://raw.githubusercontent.com/trickest/resolvers/refs/heads/main/resolvers.txt') as response:
             resolvers = await response.text()

httpz_scanner/formatters.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 # HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/formatters.py
+# httpz_scanner/formatters.py
 
 from .colors import Colors
 from .utils  import human_size
 
+
 def format_console_output(result: dict, debug: bool = False, show_fields: dict = None, match_codes: set = None, exclude_codes: set = None) -> str:
     '''
     Format the output with colored sections
@@ -37,9 +38,17 @@ def format_console_output(result: dict, debug: bool = False, show_fields: dict =
             status = f"{Colors.RED}[{result['status']}]{Colors.RESET}"
         parts.append(status)
     
-    # Domain (always shown)
+    # Domain/URL
     parts.append(f"[{result['url']}]")
     
+    # Content Type
+    if show_fields.get('content_type') and result.get('content_type'):
+        parts.append(f"{Colors.CYAN}[{result['content_type']}]{Colors.RESET}")
+    
+    # Content Length
+    if show_fields.get('content_length') and result.get('content_length'):
+        parts.append(f"{Colors.PINK}[{result['content_length']}]{Colors.RESET}")
+    
     # Title
     if show_fields.get('title') and result.get('title'):
         parts.append(f"{Colors.DARK_GREEN}[{result['title']}]{Colors.RESET}")
@@ -59,8 +68,8 @@ def format_console_output(result: dict, debug: bool = False, show_fields: dict =
         parts.append(f"{Colors.PURPLE}[{result['favicon_hash']}]{Colors.RESET}")
 
     # Headers
-    if show_fields.get('headers') and result.get('headers'):
+    if show_fields.get('headers') and result.get('response_headers'):
-        headers_text = [f"{k}: {v}" for k, v in result['headers'].items()]
+        headers_text = [f"{k}: {v}" for k, v in result['response_headers'].items()]
         parts.append(f"{Colors.CYAN}[{', '.join(headers_text)}]{Colors.RESET}")
     else:
         if show_fields.get('content_type') and result.get('content_type'):
@@ -73,17 +82,17 @@ def format_console_output(result: dict, debug: bool = False, show_fields: dict =
             except (ValueError, TypeError):
                 parts.append(f"{Colors.PINK}[{result['content_length']}]{Colors.RESET}")
 
-    # CNAME
-    if show_fields.get('cname') and result.get('cname'):
-        parts.append(f"{Colors.PURPLE}[CNAME: {result['cname']}]{Colors.RESET}")
-    
     # Redirect Chain
     if show_fields.get('follow_redirects') and result.get('redirect_chain'):
         chain = ' -> '.join(result['redirect_chain'])
         parts.append(f"{Colors.YELLOW}[Redirects: {chain}]{Colors.RESET}")
 
+    # CNAME
+    if show_fields.get('cname') and result.get('cname'):
+        parts.append(f"{Colors.PURPLE}[CNAME: {result['cname']}]{Colors.RESET}")
+    
     # TLS Certificate Info
-    if result.get('tls'):
+    if show_fields.get('tls') and result.get('tls'):
         cert = result['tls']
         tls_parts = []
         if cert.get('common_name'):

httpz_scanner/parsers.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 # HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
-# httpz/parsers.py
+# httpz_scanner/parsers.py
+
+import argparse
 
 try:
     import bs4
@@ -27,8 +29,8 @@ def parse_domain_url(domain: str) -> tuple:
     Parse domain string into base domain, port, and protocol list
     
     :param domain: Raw domain string to parse
-    :return: Tuple of (base_domain, port, protocols)
     '''
+
     port = None
     base_domain = domain.rstrip('/')
     
@@ -40,23 +42,17 @@ def parse_domain_url(domain: str) -> tuple:
             try:
                 port = int(port_str.split('/')[0])
             except ValueError:
-                port = 443 if protocol == 'https://' else 80
+                port = None
-        else:
-            port = 443 if protocol == 'https://' else 80
-        protocols = [f'{protocol}{base_domain}{":" + str(port) if port else ""}']
     else:
         if ':' in base_domain.split('/')[0]:
             base_domain, port_str = base_domain.split(':', 1)
-            port = int(port_str.split('/')[0]) if port_str.split('/')[0].isdigit() else 443
+            port = int(port_str.split('/')[0]) if port_str.split('/')[0].isdigit() else None
-        else:
+    
-            port = 443
+    protocols = ['http://', 'https://']  # Always try HTTP first
-        protocols = [
-            f'https://{base_domain}{":" + str(port) if port else ""}',
-            f'http://{base_domain}{":"  + str(port) if port else ""}'
-        ]
     
     return base_domain, port, protocols
 
+
 async def get_cert_info(ssl_object, url: str) -> dict:
     '''
     Get SSL certificate information for a domain
@@ -64,6 +60,7 @@ async def get_cert_info(ssl_object, url: str) -> dict:
     :param ssl_object: SSL object to get certificate info from
     :param url: URL to get certificate info from
     '''
+
     try:            
         if not ssl_object or not (cert_der := ssl_object.getpeercert(binary_form=True)):
             return None
@@ -100,6 +97,7 @@ async def get_cert_info(ssl_object, url: str) -> dict:
         error(f'Error getting cert info for {url}: {str(e)}')
         return None
 
+
 async def get_favicon_hash(session, base_url: str, html: str) -> str:
     '''
     Get favicon hash from a webpage
@@ -108,6 +106,7 @@ async def get_favicon_hash(session, base_url: str, html: str) -> str:
     :param base_url: base URL of the website
     :param html: HTML content of the page
     '''
+
     try:
         soup = bs4.BeautifulSoup(html, 'html.parser')
         
@@ -138,3 +137,61 @@ async def get_favicon_hash(session, base_url: str, html: str) -> str:
         debug(f'Error getting favicon for {base_url}: {str(e)}')
     
     return None 
+
+
+def parse_status_codes(codes_str: str) -> set:
+    '''
+    Parse comma-separated status codes and ranges into a set of integers
+    
+    :param codes_str: Comma-separated status codes (e.g., "200,301-399,404,500-503")
+    '''
+    
+    codes = set()
+    try:
+        for part in codes_str.split(','):
+            if '-' in part:
+                start, end = map(int, part.split('-'))
+                codes.update(range(start, end + 1))
+            else:
+                codes.add(int(part))
+        return codes
+    except ValueError:
+        raise argparse.ArgumentTypeError('Invalid status code format. Use comma-separated numbers or ranges (e.g., 200,301-399,404,500-503)')
+
+
+def parse_shard(shard_str: str) -> tuple:
+    '''
+    Parse shard argument in format INDEX/TOTAL
+    
+    :param shard_str: Shard string in format "INDEX/TOTAL"
+    '''
+
+    try:
+        shard_index, total_shards = map(int, shard_str.split('/'))
+        if shard_index < 1 or total_shards < 1 or shard_index > total_shards:
+            raise ValueError
+        return shard_index - 1, total_shards  # Convert to 0-based index
+    except (ValueError, TypeError):
+        raise argparse.ArgumentTypeError('Shard must be in format INDEX/TOTAL where INDEX <= TOTAL') 
+
+
+def parse_title(html: str, content_type: str = None) -> str:
+    '''
+    Parse title from HTML content
+    
+    :param html: HTML content of the page
+    :param content_type: Content-Type header value
+    '''
+
+    # Only parse title for HTML content
+    if content_type and not any(x in content_type.lower() for x in ['text/html', 'application/xhtml']):
+        return None
+        
+    try:
+        soup = bs4.BeautifulSoup(html, 'html.parser', from_encoding='utf-8', features='lxml')
+        if title := soup.title:
+            return title.string.strip()
+    except:
+        pass
+    
+    return None 

httpz_scanner/scanner.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+# HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
+# httpz_scanner/scanner.py
+
+import asyncio
+import random
+import urllib.parse
+import json
+
+try:
+    import aiohttp
+except ImportError:
+    raise ImportError('missing aiohttp module (pip install aiohttp)')
+
+try:
+    import bs4
+except ImportError:
+    raise ImportError('missing bs4 module (pip install beautifulsoup4)')
+
+from .dns     import resolve_all_dns, load_resolvers
+from .parsers import parse_domain_url, get_cert_info, get_favicon_hash
+from .utils   import debug, USER_AGENTS, input_generator
+
+
+class HTTPZScanner:
+    '''Core scanner class for HTTP domain checking'''
+    
+    def __init__(self, concurrent_limit = 100, timeout = 5, follow_redirects = False, check_axfr = False, resolver_file = None, output_file = None, show_progress = False, debug_mode = False, jsonl_output = False, show_fields = None, match_codes = None, exclude_codes = None, shard = None, paths = None, custom_headers=None, post_data=None):
+        '''
+        Initialize the HTTPZScanner class
+        
+        :param concurrent_limit: Maximum number of concurrent requests
+        :param timeout: Request timeout in seconds
+        :param follow_redirects: Follow redirects
+        :param check_axfr: Check for AXFR
+        :param resolver_file: Path to resolver file
+        :param output_file: Path to output file
+        :param show_progress: Show progress bar
+        :param debug_mode: Enable debug mode
+        :param jsonl_output: Output in JSONL format
+        :param show_fields: Fields to show
+        :param match_codes: Status codes to match
+        :param exclude_codes: Status codes to exclude
+        :param shard: Tuple of (shard_index, total_shards) for distributed scanning
+        :param paths: List of additional paths to check on each domain
+        :param custom_headers: Dictionary of custom headers to send with each request
+        :param post_data: Data to send with POST requests
+        '''
+
+        self.concurrent_limit = concurrent_limit
+        self.timeout          = timeout
+        self.follow_redirects = follow_redirects
+        self.check_axfr       = check_axfr
+        self.resolver_file    = resolver_file
+        self.output_file      = output_file
+        self.show_progress    = show_progress
+        self.debug_mode       = debug_mode
+        self.jsonl_output     = jsonl_output
+        self.shard            = shard
+        self.paths            = paths or []
+        self.custom_headers   = custom_headers or {}
+        self.post_data        = post_data
+
+        self.show_fields = show_fields or {
+            'status_code'      : True,
+            'content_type'     : True,
+            'content_length'   : True,
+            'title'            : True,
+            'body'             : True,
+            'ip'               : True,
+            'favicon'          : True,
+            'headers'          : True,
+            'follow_redirects' : True,
+            'cname'            : True,
+            'tls'              : True
+        }
+
+        self.match_codes       = match_codes
+        self.exclude_codes     = exclude_codes
+        self.resolvers         = None
+        self.processed_domains = 0
+        self.progress_count    = 0
+
+
+    async def check_domain(self, session: aiohttp.ClientSession, domain: str):
+        '''Check a single domain and return results'''
+        base_domain, port, protocols = parse_domain_url(domain)
+        
+        for protocol in protocols:
+            url = f'{protocol}{base_domain}'
+            if port:
+                url += f':{port}'
+                
+            try:
+                debug(f'Trying {url}...')
+                result = await self._check_url(session, url)
+                debug(f'Got result for {url}: {result}')
+                if result and (result['status'] != 400 or result.get('redirect_chain')):  # Accept redirects
+                    return result
+            except Exception as e:
+                debug(f'Error checking {url}: {str(e)}')
+                continue
+        
+        return None
+
+    async def _check_url(self, session: aiohttp.ClientSession, url: str):
+        '''Check a single URL and return results'''
+        try:
+            headers = {'User-Agent': random.choice(USER_AGENTS)}
+            headers.update(self.custom_headers)
+            
+            debug(f'Making request to {url} with headers: {headers}')
+            async with session.request('GET', url, 
+                timeout=self.timeout,
+                allow_redirects=True,  # Always follow redirects
+                max_redirects=10,
+                ssl=False,  # Don't verify SSL
+                headers=headers) as response:
+                
+                debug(f'Got response from {url}: status={response.status}, headers={dict(response.headers)}')
+                
+                result = {
+                    'domain': urllib.parse.urlparse(url).hostname,
+                    'status': response.status,
+                    'url': str(response.url),
+                    'response_headers': dict(response.headers)
+                }
+                
+                if response.history:
+                    result['redirect_chain'] = [str(h.url) for h in response.history] + [str(response.url)]
+                    debug(f'Redirect chain for {url}: {result["redirect_chain"]}')
+                
+                return result
+                
+        except aiohttp.ClientSSLError as e:
+            debug(f'SSL Error for {url}: {str(e)}')
+            return {
+                'domain': urllib.parse.urlparse(url).hostname,
+                'status': -1,
+                'error': f'SSL Error: {str(e)}',
+                'protocol': 'https' if url.startswith('https://') else 'http',
+                'error_type': 'SSL'
+            }
+        except aiohttp.ClientConnectorCertificateError as e:
+            debug(f'Certificate Error for {url}: {str(e)}')
+            return {
+                'domain': urllib.parse.urlparse(url).hostname,
+                'status': -1,
+                'error': f'Certificate Error: {str(e)}',
+                'protocol': 'https' if url.startswith('https://') else 'http',
+                'error_type': 'CERT'
+            }
+        except aiohttp.ClientConnectorError as e:
+            debug(f'Connection Error for {url}: {str(e)}')
+            return {
+                'domain': urllib.parse.urlparse(url).hostname,
+                'status': -1,
+                'error': f'Connection Failed: {str(e)}',
+                'protocol': 'https' if url.startswith('https://') else 'http',
+                'error_type': 'CONN'
+            }
+        except aiohttp.ClientError as e:
+            debug(f'HTTP Error for {url}: {e.__class__.__name__}: {str(e)}')
+            return {
+                'domain': urllib.parse.urlparse(url).hostname,
+                'status': -1,
+                'error': f'HTTP Error: {e.__class__.__name__}: {str(e)}',
+                'protocol': 'https' if url.startswith('https://') else 'http',
+                'error_type': 'HTTP'
+            }
+        except asyncio.TimeoutError:
+            debug(f'Timeout for {url}')
+            return {
+                'domain': urllib.parse.urlparse(url).hostname,
+                'status': -1,
+                'error': f'Connection Timed Out after {self.timeout}s',
+                'protocol': 'https' if url.startswith('https://') else 'http',
+                'error_type': 'TIMEOUT'
+            }
+        except Exception as e:
+            debug(f'Unexpected error for {url}: {e.__class__.__name__}: {str(e)}')
+            return {
+                'domain': urllib.parse.urlparse(url).hostname,
+                'status': -1,
+                'error': f'Error: {e.__class__.__name__}: {str(e)}',
+                'protocol': 'https' if url.startswith('https://') else 'http',
+                'error_type': 'UNKNOWN'
+            }
+
+
+    async def scan(self, input_source):
+        '''
+        Scan domains from a file, stdin, or async generator
+        
+        :param input_source: Can be:
+            - Path to file (str)
+            - stdin ('-')
+            - List/tuple of domains
+            - Async generator yielding domains
+        :yields: Result dictionary for each domain scanned
+        '''
+        
+        if not self.resolvers:
+            self.resolvers = await load_resolvers(self.resolver_file)
+
+        # Just use ssl=False, that's all we need
+        connector = aiohttp.TCPConnector(ssl=False, enable_cleanup_closed=True)
+        async with aiohttp.ClientSession(connector=connector) as session:
+            tasks = {}  # Change to dict to track domain for each task
+            domain_queue = asyncio.Queue()
+            queue_empty = False
+            
+            async def process_domain(domain):
+                try:
+                    result = await self.check_domain(session, domain)
+                    if self.show_progress:
+                        self.progress_count += 1
+                    if result:
+                        return domain, result
+                    else:
+                        # Create a proper error result if check_domain returns None
+                        return domain, {
+                            'domain': domain,
+                            'status': -1,
+                            'error': 'No successful response from either HTTP or HTTPS',
+                            'protocol': 'unknown',
+                            'error_type': 'NO_RESPONSE'
+                        }
+                except Exception as e:
+                    debug(f'Error processing {domain}: {e.__class__.__name__}: {str(e)}')
+                    # Return structured error information
+                    return domain, {
+                        'domain': domain,
+                        'status': -1,
+                        'error': f'{e.__class__.__name__}: {str(e)}',
+                        'protocol': 'unknown',
+                        'error_type': 'PROCESS'
+                    }
+
+            # Queue processor
+            async def queue_processor():
+                async for domain in input_generator(input_source, self.shard):
+                    await domain_queue.put(domain)
+                    self.processed_domains += 1
+                nonlocal queue_empty
+                queue_empty = True
+
+            # Start queue processor
+            queue_task = asyncio.create_task(queue_processor())
+
+            try:
+                while not (queue_empty and domain_queue.empty() and not tasks):
+                    # Fill up tasks until we hit concurrent limit
+                    while len(tasks) < self.concurrent_limit and not domain_queue.empty():
+                        domain = await domain_queue.get()
+                        task = asyncio.create_task(process_domain(domain))
+                        tasks[task] = domain
+                    
+                    if tasks:
+                        # Wait for at least one task to complete
+                        done, _ = await asyncio.wait(
+                            tasks.keys(),
+                            return_when=asyncio.FIRST_COMPLETED
+                        )
+                        
+                        # Process completed tasks
+                        for task in done:
+                            domain = tasks.pop(task)
+                            try:
+                                _, result = await task
+                                if result:
+                                    yield result
+                            except Exception as e:
+                                debug(f'Task error for {domain}: {e.__class__.__name__}: {str(e)}')
+                                yield {
+                                    'domain': domain,
+                                    'status': -1,
+                                    'error': f'Task Error: {e.__class__.__name__}: {str(e)}',
+                                    'protocol': 'unknown',
+                                    'error_type': 'TASK'
+                                }
+                    else:
+                        await asyncio.sleep(0.1)  # Prevent CPU spin when no tasks
+
+            finally:
+                # Clean up
+                for task in tasks:
+                    task.cancel()
+                queue_task.cancel()
+                try:
+                    await queue_task
+                except asyncio.CancelledError:
+                    pass 

httpz_scanner/utils.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# HTTPZ Web Scanner - Developed by acidvegas in Python (https://github.com/acidvegas/httpz)
+# httpz_scanner/utils.py
+
+import logging
+import os
+import sys
+import asyncio
+
+
+# Global for silent mode
+SILENT_MODE = False
+
+# List of user agents to randomize requests
+USER_AGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.6.5 Chrome/124.0.6367.243 Electron/30.1.2 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 OPR/116.0.0.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:134.0) Gecko/20100101 Firefox/134.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.8.3 Chrome/130.0.6723.191 Electron/33.3.2 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.137 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.5.12 Chrome/120.0.6099.283 Electron/28.2.3 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.0',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
+    'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 OPR/114.0.0.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.7.7 Chrome/128.0.6613.186 Electron/32.2.5 Safari/537.36'
+]
+
+
+def debug(msg: str):
+    if not SILENT_MODE:
+        logging.debug(msg)
+def error(msg: str):
+    if not SILENT_MODE: logging.error(msg)
+def info(msg: str):
+    if not SILENT_MODE: logging.info(msg)
+def warning(msg: str):
+    if not SILENT_MODE: logging.warning(msg)
+
+
+def human_size(size_bytes: int) -> str:
+    '''
+    Convert bytes to human readable string
+    
+    :param size_bytes: size in bytes
+    '''
+
+    if not size_bytes:
+        return '0B'
+    
+    units      = ('B', 'KB', 'MB', 'GB')
+    size       = float(size_bytes)
+    unit_index = 0
+    
+    while size >= 1024 and unit_index < len(units) - 1:
+        size /= 1024
+        unit_index += 1
+    
+    return f'{size:.1f}{units[unit_index]}'
+
+
+async def input_generator(input_source, shard: tuple = None):
+    '''
+    Async generator function to yield domains from various input sources with optional sharding
+    
+    :param input_source: Can be:
+        - string path to local file
+        - "-" for stdin
+        - list/tuple of domains
+        - generator/iterator yielding domains
+        - string content with newlines
+    :param shard: Tuple of (shard_index, total_shards) for distributed scanning
+    '''
+    
+    line_num = 0
+    
+    # Handle stdin
+    if input_source == '-' or input_source is None:
+        for line in sys.stdin:
+            await asyncio.sleep(0)
+            if line := line.strip():
+                if shard is None or line_num % shard[1] == shard[0]:
+                    yield line
+            line_num += 1
+            
+    # Handle local files
+    elif isinstance(input_source, str) and os.path.exists(input_source):
+        with open(input_source, 'r') as f:
+            for line in f:
+                await asyncio.sleep(0)
+                if line := line.strip():
+                    if shard is None or line_num % shard[1] == shard[0]:
+                        yield line
+                line_num += 1
+                
+    # Handle iterables (generators, lists, etc)
+    elif hasattr(input_source, '__iter__') and not isinstance(input_source, (str, bytes)):
+        for line in input_source:
+            await asyncio.sleep(0)
+            if isinstance(line, bytes):
+                line = line.decode()
+            if line := line.strip():
+                if shard is None or line_num % shard[1] == shard[0]:
+                    yield line
+            line_num += 1
+            
+    # Handle string content with newlines
+    elif isinstance(input_source, (str, bytes)):
+        if isinstance(input_source, bytes):
+            input_source = input_source.decode()
+        for line in input_source.splitlines():
+            await asyncio.sleep(0)
+            if line := line.strip():
+                if shard is None or line_num % shard[1] == shard[0]:
+                    yield line
+            line_num += 1

setup.py

@@ -4,12 +4,13 @@
 
 from setuptools import setup, find_packages
 
+
 with open('README.md', 'r', encoding='utf-8') as f:
     long_description = f.read()
 
 setup(
-    name='httpz-scanner',
+    name='httpz_scanner',
-    version='1.0.3',
+    version='2.1.8',
     author='acidvegas',
     author_email='acid.vegas@acid.vegas',
     description='Hyper-fast HTTP Scraping Tool',
@@ -39,4 +40,9 @@ setup(
         'dnspython>=2.1.0',
         'mmh3>=3.0.0',
     ],
+    entry_points={
+        'console_scripts': [
+            'httpz=httpz_scanner.cli:run',
+        ],
+    },
 ) 

unit_test.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+# HTTPZ Web Scanner - Unit Tests
+# unit_test.py
+
+import asyncio
+import logging
+import sys
+import time
+
+try:
+    from httpz_scanner        import HTTPZScanner
+    from httpz_scanner.colors import Colors
+except ImportError:
+    raise ImportError('missing httpz_scanner library (pip install httpz_scanner)')
+
+
+class ColoredFormatter(logging.Formatter):
+    '''Custom formatter for colored log output'''
+    
+    def format(self, record):
+        if record.levelno == logging.INFO:
+            color = Colors.GREEN
+        elif record.levelno == logging.WARNING:
+            color = Colors.YELLOW
+        elif record.levelno == logging.ERROR:
+            color = Colors.RED
+        else:
+            color = Colors.RESET
+            
+        record.msg = f'{color}{record.msg}{Colors.RESET}'
+        return super().format(record)
+
+
+# Configure logging with colors
+logger = logging.getLogger()
+handler = logging.StreamHandler()
+handler.setFormatter(ColoredFormatter('%(asctime)s - %(levelname)s - %(message)s'))
+logger.setLevel(logging.INFO)
+logger.addHandler(handler)
+
+
+async def get_domains_from_url() -> list:
+    '''
+    Fetch domains from SecLists URL
+    
+    :return: List of domains
+    '''
+    
+    try:
+        import aiohttp
+    except ImportError:
+        raise ImportError('missing aiohttp library (pip install aiohttp)')
+
+    url = 'https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Fuzzing/email-top-100-domains.txt'
+    
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url) as response:
+            content = await response.text()
+            return [line.strip() for line in content.splitlines() if line.strip()]
+
+
+async def domain_generator(domains: list):
+    '''
+    Async generator that yields domains
+    
+    :param domains: List of domains to yield
+    '''
+    
+    for domain in domains:
+        await asyncio.sleep(0) # Allow other coroutines to run
+        yield domain
+
+
+async def run_benchmark(test_type: str, domains: list, concurrency: int) -> tuple:
+    '''Run a single benchmark test'''
+    
+    logging.info(f'{Colors.BOLD}Testing {test_type} input with {concurrency} concurrent connections...{Colors.RESET}')
+    scanner = HTTPZScanner(concurrent_limit=concurrency, timeout=3, show_progress=True, debug_mode=True, follow_redirects=True)
+    
+    count = 0
+    got_first = False
+    start_time = None
+    
+    if test_type == 'List':
+        async for result in scanner.scan(domains):
+            if result:
+                if not got_first:
+                    got_first = True
+                    start_time = time.time()
+                count += 1
+                
+                # More detailed status reporting
+                status_str = ''
+                if result['status'] < 0:
+                    error_type = result.get('error_type', 'UNKNOWN')
+                    error_msg = result.get('error', 'Unknown Error')
+                    status_str = f"{Colors.RED}[{result['status']} - {error_type}: {error_msg}]{Colors.RESET}"
+                elif 200 <= result['status'] < 300:
+                    status_str = f"{Colors.GREEN}[{result['status']}]{Colors.RESET}"
+                elif 300 <= result['status'] < 400:
+                    status_str = f"{Colors.YELLOW}[{result['status']}]{Colors.RESET}"
+                else:
+                    status_str = f"{Colors.RED}[{result['status']}]{Colors.RESET}"
+                
+                # Show protocol and response headers if available
+                protocol_info = f" {Colors.CYAN}({result.get('protocol', 'unknown')}){Colors.RESET}" if result.get('protocol') else ''
+                headers_info = ''
+                if result.get('response_headers'):
+                    important_headers = ['server', 'location', 'content-type']
+                    headers = [f"{k}: {v}" for k, v in result['response_headers'].items() if k.lower() in important_headers]
+                    if headers:
+                        headers_info = f" {Colors.GRAY}[{', '.join(headers)}]{Colors.RESET}"
+                
+                # Show redirect chain if present
+                redirect_info = ''
+                if result.get('redirect_chain'):
+                    redirect_info = f" -> {Colors.YELLOW}Redirects: {' -> '.join(result['redirect_chain'])}{Colors.RESET}"
+                
+                # Show error details if present
+                error_info = ''
+                if result.get('error'):
+                    error_info = f" {Colors.RED}Error: {result['error']}{Colors.RESET}"
+                
+                # Show final URL if different from original
+                url_info = ''
+                if result.get('url') and result['url'] != f"http(s)://{result['domain']}":
+                    url_info = f" {Colors.CYAN}Final URL: {result['url']}{Colors.RESET}"
+                
+                logging.info(
+                    f"{test_type}-{concurrency} Result {count}: "
+                    f"{status_str}{protocol_info} "
+                    f"{Colors.CYAN}{result['domain']}{Colors.RESET}"
+                    f"{redirect_info}"
+                    f"{url_info}"
+                    f"{headers_info}"
+                    f"{error_info}"
+                )
+    else:
+        # Skip generator test
+        pass
+
+    elapsed = time.time() - start_time if start_time else 0
+    domains_per_sec = count/elapsed if elapsed > 0 else 0
+    logging.info(f'{Colors.YELLOW}{test_type} test with {concurrency} concurrent connections completed in {elapsed:.2f} seconds ({domains_per_sec:.2f} domains/sec){Colors.RESET}')
+    
+    return elapsed, domains_per_sec
+
+
+async def test_list_input(domains: list):
+    '''Test scanning using a list input'''
+    
+    logging.info(f'{Colors.BOLD}Testing list input...{Colors.RESET}')
+    scanner = HTTPZScanner(concurrent_limit=25, timeout=3, show_progress=True, debug_mode=True, follow_redirects=True)
+    
+    start_time = time.time()
+    count = 0
+    async for result in scanner.scan(domains):
+        if result:
+            count += 1
+            status_color = Colors.GREEN if 200 <= result['status'] < 300 else Colors.RED
+            title = f" - {Colors.CYAN}{result.get('title', 'No Title')}{Colors.RESET}" if result.get('title') else ''
+            error = f" - {Colors.RED}{result.get('error', '')}{Colors.RESET}" if result.get('error') else ''
+            logging.info(f'List-25 Result {count}: {status_color}[{result["status"]}]{Colors.RESET} {Colors.CYAN}{result["domain"]}{Colors.RESET}{title}{error}')
+
+
+async def test_generator_input(domains: list):
+    '''Test scanning using an async generator input'''
+    
+    logging.info(f'{Colors.BOLD}Testing generator input...{Colors.RESET}')
+    scanner = HTTPZScanner(concurrent_limit=25, timeout=3, show_progress=True, debug_mode=True, follow_redirects=True)
+    
+    start_time = time.time()
+    count = 0
+    async for result in scanner.scan(domain_generator(domains)):
+        if result:
+            count += 1
+            status_color = Colors.GREEN if 200 <= result['status'] < 300 else Colors.RED
+            title = f" - {Colors.CYAN}{result.get('title', 'No Title')}{Colors.RESET}" if result.get('title') else ''
+            error = f" - {Colors.RED}{result.get('error', '')}{Colors.RESET}" if result.get('error') else ''
+            logging.info(f'Generator-25 Result {count}: {status_color}[{result["status"]}]{Colors.RESET} {Colors.CYAN}{result["domain"]}{Colors.RESET}{title}{error}')
+
+
+async def main() -> None:
+    '''Main test function'''
+    
+    try:
+        # Fetch domains
+        domains = await get_domains_from_url()
+        logging.info(f'Loaded {Colors.YELLOW}{len(domains)}{Colors.RESET} domains for testing')
+        
+        # Store benchmark results
+        results = []
+        
+        # Run tests with different concurrency levels
+        for concurrency in [25, 50, 100]:
+            # Generator tests
+            gen_result = await run_benchmark('Generator', domains, concurrency)
+            results.append(('Generator', concurrency, *gen_result))
+            
+            # List tests
+            list_result = await run_benchmark('List', domains, concurrency)
+            results.append(('List', concurrency, *list_result))
+        
+        # Print benchmark comparison
+        logging.info(f'\n{Colors.BOLD}Benchmark Results:{Colors.RESET}')
+        logging.info('-' * 80)
+        logging.info(f'{"Test Type":<15} {"Concurrency":<15} {"Time (s)":<15} {"Domains/sec":<15}')
+        logging.info('-' * 80)
+        
+        # Sort by domains per second (fastest first)
+        results.sort(key=lambda x: x[3], reverse=True)
+        
+        for test_type, concurrency, elapsed, domains_per_sec in results:
+            logging.info(f'{test_type:<15} {concurrency:<15} {elapsed:.<15.2f} {domains_per_sec:<15.2f}')
+        
+        # Highlight fastest result
+        fastest = results[0]
+        logging.info('-' * 80)
+        logging.info(f'{Colors.GREEN}Fastest: {fastest[0]} test with {fastest[1]} concurrent connections')
+        logging.info(f'Time: {fastest[2]:.2f} seconds')
+        logging.info(f'Speed: {fastest[3]:.2f} domains/sec{Colors.RESET}')
+        
+        logging.info(f'\n{Colors.GREEN}All tests completed successfully!{Colors.RESET}')
+        
+    except Exception as e:
+        logging.error(f'Test failed: {Colors.RED}{str(e)}{Colors.RESET}')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        logging.warning(f'{Colors.YELLOW}Tests interrupted by user{Colors.RESET}')
+        sys.exit(1)