Compare commits

..

No commits in common. "main" and "v1.1.0" have entirely different histories.
main ... v1.1.0

15 changed files with 211 additions and 318 deletions

3
.gitignore vendored
View File

@ -4,5 +4,4 @@ logs/*
*.log
.log.
*.egg-info
dist/
zones/
dist/

View File

@ -28,22 +28,19 @@ czds [-h] [-u USERNAME] [-p PASSWORD] [-z] [-c CONCURRENCY] [-d] [-k] [-r] [-s]
```
#### Arguments
###### Basic Options
| Argument | Description | Default |
|-----------------------|----------------------------------------------|-------------------|
| `-h`, `--help` | Show help message and exit | |
| `-u`, `--username` | ICANN Username | `$CZDS_USER` |
| `-p`, `--password` | ICANN Password | `$CZDS_PASS` |
| `-o`, `--output` | Output directory | Current directory |
###### Zone Options
| `-z`, `--zones` | Download zone files | |
| `-c`, `--concurrency` | Number of concurrent downloads | `3` |
###### Report Options
| `-d`, `--decompress` | Decompress zone files after download | |
| `-k`, `--keep` | Keep original gzip files after decompression | |
| `-r`, `--report` | Download the zone stats report | |
| `-s`, `--scrub` | Scrub username from the report | |
| `-f`, `--format` | Report output format (csv/json) | `csv` |
| `-o`, `--output` | Output directory | Current directory |
### Environment Variables
```bash

View File

@ -4,7 +4,8 @@
from .client import CZDS
__version__ = '1.3.8'
__version__ = '1.1.0'
__author__ = 'acidvegas'
__email__ = 'acid.vegas@acid.vegas'
__github__ = 'https://github.com/acidvegas/czds'

View File

@ -7,6 +7,7 @@ import asyncio
import getpass
import logging
import os
import time
from .client import CZDS
@ -20,23 +21,24 @@ async def main():
# Authentication
parser.add_argument('-u', '--username', default=os.getenv('CZDS_USER'), help='ICANN Username')
parser.add_argument('-p', '--password', default=os.getenv('CZDS_PASS'), help='ICANN Password')
parser.add_argument('-o', '--output', default=os.getcwd(), help='Output directory')
# Zone download options
zone_group = parser.add_argument_group('Zone download options')
zone_group.add_argument('-z', '--zones', action='store_true', help='Download zone files')
zone_group.add_argument('-c', '--concurrency', type=int, default=3, help='Number of concurrent downloads')
parser.add_argument('-z', '--zones', action='store_true', help='Download zone files')
parser.add_argument('-c', '--concurrency', type=int, default=3, help='Number of concurrent downloads')
parser.add_argument('-d', '--decompress', action='store_true', help='Decompress zone files after download')
parser.add_argument('-k', '--keep', action='store_true', help='Keep the original gzip files after decompression')
# Report options
report_group = parser.add_argument_group('Report options')
report_group.add_argument('-r', '--report', action='store_true', help='Download the zone stats report')
report_group.add_argument('-s', '--scrub', action='store_true', help='Scrub the username from the report')
report_group.add_argument('-f', '--format', choices=['csv', 'json'], default='csv', help='Report output format')
parser.add_argument('-r', '--report', action='store_true', help='Download the zone stats report')
parser.add_argument('-s', '--scrub', action='store_true', help='Scrub the username from the report')
parser.add_argument('-f', '--format', choices=['csv', 'json'], default='csv', help='Report output format')
# Output options
parser.add_argument('-o', '--output', default=os.getcwd(), help='Output directory')
# Parse arguments
args = parser.parse_args()
# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# Get username and password
@ -44,43 +46,34 @@ async def main():
password = args.password or getpass.getpass('ICANN Password: ')
# Create output directory
output_directory = os.path.join(args.output, 'zones')
now = time.strftime('%Y-%m-%d')
output_directory = os.path.join(args.output, 'zones', now)
os.makedirs(output_directory, exist_ok=True)
logging.info('Authenticating with ICANN API...')
# Create the CZDS client
async with CZDS(username, password) as client:
# Download zone stats report if requested
if args.report:
logging.info('Fetching zone stats report...')
try:
# Create the report directory
output = os.path.join(output_directory, '.report.csv')
# Download the report
await client.get_report(output, scrub=args.scrub, format=args.format)
logging.info(f'Zone stats report saved to {output}')
return
except Exception as e:
raise Exception(f'Failed to download zone stats report: {e}')
# Download zone files if requested
if args.zones:
logging.info('Downloading zone files...')
logging.info('Fetching zone links...')
try:
# Download the zone files
await client.download_zones(output_directory, args.concurrency)
zone_links = await client.fetch_zone_links()
except Exception as e:
raise Exception(f'Failed to download zone files: {e}')
raise Exception(f'Failed to fetch zone links: {e}')
def cli_entry():
'''Synchronous entry point for console script'''
return asyncio.run(main())
logging.info(f'Downloading {len(zone_links):,} zone files...')
await client.download_zones(zone_links, output_directory, args.concurrency, decompress=args.decompress, cleanup=not args.keep)

View File

@ -3,11 +3,8 @@
# czds/client.py
import asyncio
import json
import logging
import os
import csv
import io
import gzip
try:
import aiohttp
@ -19,17 +16,6 @@ try:
except ImportError:
raise ImportError('missing aiofiles library (pip install aiofiles)')
try:
from tqdm import tqdm
except ImportError:
raise ImportError('missing tqdm library (pip install tqdm)')
from .utils import gzip_decompress, humanize_bytes
# Configure logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
class CZDS:
'''Class for the ICANN Centralized Zones Data Service'''
@ -42,37 +28,17 @@ class CZDS:
:param password: ICANN Password
'''
# Set the username and password
self.username = username
self.password = password
# Configure TCP keepalive
connector = aiohttp.TCPConnector(
keepalive_timeout=300, # Keep connections alive for 5 minutes
force_close=False, # Don't force close connections
enable_cleanup_closed=True, # Cleanup closed connections
ttl_dns_cache=300, # Cache DNS results for 5 minutes
)
# Set the session with longer timeouts and keepalive
self.session = aiohttp.ClientSession(
connector=connector,
timeout=aiohttp.ClientTimeout(total=None, connect=60, sock_connect=60, sock_read=None),
headers={'Connection': 'keep-alive'},
raise_for_status=True
)
# Placeholder for the headers after authentication
self.headers = None
logging.info('Initialized CZDS client')
self.headers = None # Store the authorization header for reuse
self.session = None # Store the client session for reuse
async def __aenter__(self):
'''Async context manager entry'''
# Authenticate with the ICANN API
await self.authenticate()
self.session = aiohttp.ClientSession()
self.headers = {'Authorization': f'Bearer {await self.authenticate()}'}
return self
@ -80,240 +46,149 @@ class CZDS:
async def __aexit__(self, exc_type, exc_val, exc_tb):
'''Async context manager exit'''
# Close the client session
await self.close()
async def close(self):
'''Close the client session'''
# Close the client session if it exists
if self.session:
await self.session.close()
logging.debug('Closed aiohttp session')
async def authenticate(self) -> str:
'''Authenticate with the ICANN API and return the access token'''
# Set the data to be sent to the API
data = {'username': self.username, 'password': self.password}
try:
data = {'username': self.username, 'password': self.password}
logging.info('Authenticating with ICANN API...')
async with self.session.post('https://account-api.icann.org/api/authenticate', json=data) as response:
if response.status != 200:
raise Exception(f'Authentication failed: {response.status} {await response.text()}')
# Send the request to the API
async with self.session.post('https://account-api.icann.org/api/authenticate', json=data) as response:
if response.status != 200:
raise Exception(f'Authentication failed: {response.status} {await response.text()}')
result = await response.json()
# Get the result from the API
result = await response.json()
return result['accessToken']
logging.info('Successfully authenticated with ICANN API')
# Set the headers for the API requests
self.headers = {'Authorization': f'Bearer {result["accessToken"]}'}
return result['accessToken']
except Exception as e:
raise Exception(f'Failed to authenticate with ICANN API: {e}')
async def fetch_zone_links(self) -> list:
'''Fetch the list of zone files available for download'''
logging.info('Fetching zone file links...')
# Send the request to the API
async with self.session.get('https://czds-api.icann.org/czds/downloads/links', headers=self.headers) as response:
if response.status != 200:
raise Exception(f'Failed to fetch zone links: {response.status} {await response.text()}')
# Get the result from the API
links = await response.json()
logging.info(f'Successfully fetched {len(links):,} zone links')
return links
return await response.json()
async def get_report(self, filepath: str = None, format: str = 'csv') -> str | dict:
async def get_report(self, filepath: str = None, scrub: bool = True, format: str = 'csv') -> str | dict:
'''
Downloads the zone report stats from the API and scrubs the report for privacy
:param filepath: Filepath to save the scrubbed report
:param scrub: Whether to scrub the username from the report
:param format: Output format ('csv' or 'json')
:return: Report content as CSV string or JSON dict
'''
logging.info('Downloading zone stats report')
# Send the request to the API
async with self.session.get('https://czds-api.icann.org/czds/requests/report', headers=self.headers) as response:
if response.status != 200:
raise Exception(f'Failed to download the zone stats report: {response.status} {await response.text()}')
# Get the content of the report
content = await response.text()
# Scrub the username from the report
content = content.replace(self.username, 'nobody@no.name')
logging.debug('Scrubbed username from report')
if scrub:
content = content.replace(self.username, 'nobody@no.name')
# Convert the report to JSON format if requested
if format.lower() == 'json':
# Parse CSV content
csv_reader = csv.DictReader(io.StringIO(content))
# Convert to list of dicts with formatted keys
json_data = []
for row in csv_reader:
formatted_row = {
key.lower().replace(' ', '_'): value
for key, value in row.items()
}
json_data.append(formatted_row)
content = json.dumps(json_data, indent=4)
logging.debug('Converted report to JSON format')
rows = [row.split(',') for row in content.strip().split('\n')]
header = rows[0]
content = [dict(zip(header, row)) for row in rows[1:]]
# Save the report to a file if a filepath is provided
if filepath:
async with aiofiles.open(filepath, 'w') as file:
await file.write(content)
logging.info(f'Saved report to {filepath}')
if format.lower() == 'json':
import json
await file.write(json.dumps(content, indent=4))
else:
await file.write(content)
return content
async def gzip_decompress(self, filepath: str, cleanup: bool = True):
'''
Decompress a gzip file in place
:param filepath: Path to the gzip file
:param cleanup: Whether to remove the original gzip file after decompression
'''
output_path = filepath[:-3] # Remove .gz extension
async with aiofiles.open(filepath, 'rb') as f_in:
content = await f_in.read()
with gzip.open(content, 'rb') as gz:
async with aiofiles.open(output_path, 'wb') as f_out:
await f_out.write(gz.read())
if cleanup:
os.remove(filepath)
async def download_zone(self, url: str, output_directory: str, semaphore: asyncio.Semaphore):
async def download_zone(self, url: str, output_directory: str, decompress: bool = False, cleanup: bool = True, semaphore: asyncio.Semaphore = None):
'''
Download a single zone file
:param url: URL to download
:param output_directory: Directory to save the zone file
:param decompress: Whether to decompress the gzip file after download
:param cleanup: Whether to remove the original gzip file after decompression
:param semaphore: Optional semaphore for controlling concurrency
'''
async def _download():
tld_name = url.split('/')[-1].split('.')[0] # Extract TLD from URL
max_retries = 20 # Maximum number of retries for failed downloads
retry_delay = 5 # Delay between retries in seconds
# Headers for better connection stability
download_headers = {
**self.headers,
'Connection': 'keep-alive',
'Keep-Alive': 'timeout=600', # 10 minutes
'Accept-Encoding': 'gzip'
}
async with self.session.get(url, headers=self.headers) as response:
if response.status != 200:
raise Exception(f'Failed to download {url}: {response.status} {await response.text()}')
# Start the attempt loop
for attempt in range(max_retries):
try:
logging.info(f'Starting download of {tld_name} zone file{" (attempt " + str(attempt + 1) + ")" if attempt > 0 else ""}')
if not (content_disposition := response.headers.get('Content-Disposition')):
raise ValueError('Missing Content-Disposition header')
# Send the request to the API
async with self.session.get(url, headers=download_headers) as response:
# Check if the request was successful
if response.status != 200:
logging.error(f'Failed to download {tld_name}: {response.status} {await response.text()}')
filename = content_disposition.split('filename=')[-1].strip('"')
filepath = os.path.join(output_directory, filename)
# Retry the download if there are more attempts
if attempt + 1 < max_retries:
logging.info(f'Retrying {tld_name} in {retry_delay:,} seconds...')
await asyncio.sleep(retry_delay)
continue
async with aiofiles.open(filepath, 'wb') as file:
while True:
chunk = await response.content.read(8192)
if not chunk:
break
await file.write(chunk)
raise Exception(f'Failed to download {tld_name}: {response.status} {await response.text()}')
if decompress:
await self.gzip_decompress(filepath, cleanup)
filepath = filepath[:-3] # Remove .gz extension
# Get expected file size from headers
if not (expected_size := int(response.headers.get('Content-Length', 0))):
raise ValueError(f'Missing Content-Length header for {tld_name}')
return filepath
# Check if the Content-Disposition header is present
if not (content_disposition := response.headers.get('Content-Disposition')):
raise ValueError(f'Missing Content-Disposition header for {tld_name}')
# Extract the filename from the Content-Disposition header
filename = content_disposition.split('filename=')[-1].strip('"')
# Create the filepath
filepath = os.path.join(output_directory, filename)
# Create a progress bar to track the download
with tqdm(total=expected_size, unit='B', unit_scale=True, desc=f'Downloading {tld_name}', leave=False) as pbar:
# Open the file for writing
async with aiofiles.open(filepath, 'wb') as file:
# Initialize the total size for tracking
total_size = 0
# Write the chunk to the file
try:
async for chunk in response.content.iter_chunked(8192):
await file.write(chunk)
total_size += len(chunk)
pbar.update(len(chunk))
except Exception as e:
logging.error(f'Connection error while downloading {tld_name}: {str(e)}')
if attempt + 1 < max_retries:
logging.info(f'Retrying {tld_name} in {retry_delay} seconds...')
await asyncio.sleep(retry_delay)
continue
raise
# Verify file size
if expected_size and total_size != expected_size:
error_msg = f'Incomplete download for {tld_name}: Got {humanize_bytes(total_size)}, expected {humanize_bytes(expected_size)}'
logging.error(error_msg)
os.remove(filepath)
if attempt + 1 < max_retries:
logging.info(f'Retrying {tld_name} in {retry_delay} seconds...')
await asyncio.sleep(retry_delay)
continue
raise Exception(error_msg)
logging.info(f'Successfully downloaded {tld_name} zone file ({humanize_bytes(total_size)})')
await gzip_decompress(filepath)
filepath = filepath[:-3]
logging.info(f'Decompressed {tld_name} zone file')
return filepath
except Exception as e:
if attempt + 1 >= max_retries:
logging.error(f'Failed to download {tld_name} after {max_retries} attempts: {str(e)}')
if 'filepath' in locals() and os.path.exists(filepath):
os.remove(filepath)
raise
logging.warning(f'Download attempt {attempt + 1} failed for {tld_name}: {str(e)}')
await asyncio.sleep(retry_delay)
async with semaphore:
if semaphore:
async with semaphore:
return await _download()
else:
return await _download()
async def download_zones(self, output_directory: str, concurrency: int):
async def download_zones(self, zone_links: list, output_directory: str, concurrency: int, decompress: bool = False, cleanup: bool = True):
'''
Download multiple zone files concurrently
:param zone_links: List of zone URLs to download
:param output_directory: Directory to save the zone files
:param concurrency: Number of concurrent downloads
:param decompress: Whether to decompress the gzip files after download
:param cleanup: Whether to remove the original gzip files after decompression
'''
# Create the output directory if it doesn't exist
os.makedirs(output_directory, exist_ok=True)
# Get the zone links
zone_links = await self.fetch_zone_links()
zone_links.sort() # Sort the zone alphabetically for better tracking
# Create a semaphore to limit the number of concurrent downloads
semaphore = asyncio.Semaphore(concurrency)
tasks = [self.download_zone(url, output_directory, decompress, cleanup, semaphore) for url in zone_links]
logging.info(f'Downloading {len(zone_links):,} zone files...')
# Create a list of tasks to download the zone files
tasks = [self.download_zone(url, output_directory, semaphore) for url in zone_links]
# Run the tasks concurrently
await asyncio.gather(*tasks)
logging.info(f'Completed downloading {len(zone_links):,} zone files')

View File

@ -1,76 +0,0 @@
#!/usr/bin/env python3
# ICANN API for the Centralized Zones Data Service - developed by acidvegas (https://git.acid.vegas/czds)
# czds/utils.py
import asyncio
import gzip
import logging
import os
try:
import aiofiles
except ImportError:
raise ImportError('missing aiofiles library (pip install aiofiles)')
try:
from tqdm import tqdm
except ImportError:
raise ImportError('missing tqdm library (pip install tqdm)')
async def gzip_decompress(filepath: str, cleanup: bool = True):
'''
Decompress a gzip file in place
:param filepath: Path to the gzip file
:param cleanup: Whether to remove the original gzip file after decompressions
'''
original_size = os.path.getsize(filepath)
output_path = filepath[:-3]
logging.debug(f'Decompressing {filepath} ({humanize_bytes(original_size)})...')
# Use a large chunk size (256MB) for maximum throughput
chunk_size = 256 * 1024 * 1024
# Run the actual decompression in a thread pool to prevent blocking
with tqdm(total=original_size, unit='B', unit_scale=True, desc=f'Decompressing {os.path.basename(filepath)}', leave=False) as pbar:
async with aiofiles.open(output_path, 'wb') as f_out:
# Run gzip decompression in thread pool since it's CPU-bound
loop = asyncio.get_event_loop()
with gzip.open(filepath, 'rb') as gz:
while True:
chunk = await loop.run_in_executor(None, gz.read, chunk_size)
if not chunk:
break
await f_out.write(chunk)
pbar.update(len(chunk))
decompressed_size = os.path.getsize(output_path)
logging.debug(f'Decompressed {filepath} ({humanize_bytes(decompressed_size)})')
if cleanup:
os.remove(filepath)
logging.debug(f'Removed original gzip file: {filepath}')
def humanize_bytes(bytes: int) -> str:
'''
Humanize a number of bytes
:param bytes: The number of bytes to humanize
'''
# List of units
units = ('B','KB','MB','GB','TB','PB','EB','ZB','YB')
# Iterate over the units
for unit in units:
# If the bytes are less than 1024, return the bytes with the unit
if bytes < 1024:
return f'{bytes:.2f} {unit}' if unit != 'B' else f'{bytes} {unit}'
# Divide the bytes by 1024
bytes /= 1024
return f'{bytes:.2f} {units[-1]}'

View File

@ -0,0 +1,90 @@
Metadata-Version: 2.2
Name: czds-api
Version: 1.0.1
Summary: ICANN API for the Centralized Zones Data Service
Home-page: https://github.com/acidvegas/czds
Author: acidvegas
Author-email: acid.vegas@acid.vegas
Project-URL: Bug Tracker, https://github.com/acidvegas/czds/issues
Project-URL: Documentation, https://github.com/acidvegas/czds#readme
Project-URL: Source Code, https://github.com/acidvegas/czds
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: ISC License (ISCL)
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Topic :: Internet
Classifier: Topic :: Security
Classifier: Topic :: Software Development :: Libraries :: Python Modules
Requires-Python: >=3.6
Description-Content-Type: text/markdown
License-File: LICENSE
Dynamic: author
Dynamic: author-email
Dynamic: classifier
Dynamic: description
Dynamic: description-content-type
Dynamic: home-page
Dynamic: project-url
Dynamic: requires-python
Dynamic: summary
# ICANN Centralized Zone Data Service API
The [ICANN Centralized Zone Data Service](https://czds.icann.org) *(CZDS)* allows *approved* users to request and download DNS zone files in bulk, provided they represent a legitimate company or academic institution and their intended use is legal and ethical. Once ICANN approves the request, this tool streamlines the retrieval of extensive domain name system data, facilitating research and security analysis in the realm of internet infrastructure.
## Zone Information
Zone files are updated once every 24 hours, specifically from 00:00 UTC to 06:00 UTC. Access to these zones is granted in increments, and the total time for approval across all zones may extend to a month or longer. It is typical for more than 90% of requested zones to receive approval. Access to certain zone files may require additional application forms with the TLD organization. Please be aware that access to certain zones is time-bound, expiring at the beginning of the following year, or up to a decade after the initial approval has been confirmed.
At the time of writing this repository, the CZDS offers access to 1,151 zones in total.
1,079 have been approved, 55 are still pending *(after 3 months)*, 10 have been revoked because the TLDs are longer active, and 6 have been denied. Zones that have expired automatically had the expiration extended for me without doing anything, aside from 13 zones that remained expired. I have included a recent [stats file](./extras/stats.csv) directly from my ICANN account.
## Installation
```bash
pip install czds-api
```
## Usage
###### Command line
```bash
czds [--username <username> --password <password>] [--concurrency <int>]
```
You can also set the `CZDS_USER` & `CZDS_PASS` environment variables to automatically authenticate:
```bash
export CZDS_USER='your_username'
export CZDS_PASS='your_password'
```
###### As a Python module
```python
import os
from czds import CZDS
CZDS_client = CZDS(username, password)
CZDS_client.download_report('report.csv')
zone_links = CZDS_client.fetch_zone_links()
os.makedirs('zones', exist_ok=True)
for zone_link in zone_links:
CZDS_client.download_zone(zone_link, 'zones')
```
## Respects & extras
While ICANN does have an official [czds-api-client-python](https://github.com/icann/czds-api-client-python) repository, I rewrote it from scratch to be more streamline & included a [POSIX version](./extras/czds) for portability. There is some [official documentation](https://raw.githubusercontent.com/icann/czds-api-client-java/master/docs/ICANN_CZDS_api.pdf) that was referenced in the creation of the POSIX version. Either way, big props to ICANN for allowing me to use the CZDS for research purposes!
___
###### Mirrors for this repository: [acid.vegas](https://git.acid.vegas/czds) • [SuperNETs](https://git.supernets.org/acidvegas/czds) • [GitHub](https://github.com/acidvegas/czds) • [GitLab](https://gitlab.com/acidvegas/czds) • [Codeberg](https://codeberg.org/acidvegas/czds)

View File

@ -0,0 +1,11 @@
LICENSE
README.md
setup.py
czds/__init__.py
czds/__main__.py
czds/client.py
czds_api.egg-info/PKG-INFO
czds_api.egg-info/SOURCES.txt
czds_api.egg-info/dependency_links.txt
czds_api.egg-info/entry_points.txt
czds_api.egg-info/top_level.txt

View File

@ -0,0 +1 @@

View File

@ -0,0 +1,2 @@
[console_scripts]
czds = czds.__main__:main

View File

@ -0,0 +1 @@
czds

BIN
dist/czds_api-1.0.0-py3-none-any.whl vendored Normal file

Binary file not shown.

BIN
dist/czds_api-1.0.0.tar.gz vendored Normal file

Binary file not shown.

View File

@ -1,3 +1,2 @@
aiohttp
aiofiles
tqdm
aiofiles

View File

@ -11,7 +11,7 @@ with open('README.md', 'r', encoding='utf-8') as fh:
setup(
name='czds-api',
version='1.3.8',
version='1.1.0',
author='acidvegas',
author_email='acid.vegas@acid.vegas',
description='ICANN API for the Centralized Zones Data Service',
@ -43,7 +43,7 @@ setup(
python_requires='>=3.6',
entry_points={
'console_scripts': [
'czds=czds.__main__:cli_entry',
'czds=czds.__main__:main',
],
},
install_requires=[