acidvegas/czds
acidvegas/czds
1
Fork 0
Code Issues Pull Requests Releases 2 Activity

version updated

Browse Source
This commit is contained in:
Dionysus 2025-03-21 21:22:57 -04:00
parent aea3ae6a6d
commit 6585a7b081
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
3 changed files with 84 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
czds/__init__.py
View File

@ -5,7 +5,7 @@
from .client import CZDS from .client import CZDS
__version__ = '1.2.8' __version__ = '1.2.9'
__author__ = 'acidvegas' __author__ = 'acidvegas'
__email__ = 'acid.vegas@acid.vegas' __email__ = 'acid.vegas@acid.vegas'
__github__ = 'https://github.com/acidvegas/czds' __github__ = 'https://github.com/acidvegas/czds'

132
czds/client.py
View File

@ -36,8 +36,11 @@ class CZDS:
self.username = username self.username = username
self.password = password self.password = password
self.session = aiohttp.ClientSession()
self.headers = None # Configure longer timeouts and proper SSL settings
timeout = aiohttp.ClientTimeout(total=None, connect=60, sock_connect=60, sock_read=60)
self.session = aiohttp.ClientSession(timeout=timeout)
self.headers = None
logging.info('Initialized CZDS client') logging.info('Initialized CZDS client')
@ -186,73 +189,100 @@ class CZDS:
async def _download(): async def _download():
tld = url.split('/')[-1].split('.')[0] # Extract TLD from URL tld = url.split('/')[-1].split('.')[0] # Extract TLD from URL
logging.info(f'Starting download of {tld} zone file') max_retries = 3
retry_delay = 5 # seconds
try: for attempt in range(max_retries):
async with self.session.get(url, headers=self.headers) as response: try:
if response.status != 200: logging.info(f'Starting download of {tld} zone file{" (attempt " + str(attempt + 1) + ")" if attempt > 0 else ""}')
error_msg = f'Failed to download {tld}: {response.status} {await response.text()}'
logging.error(error_msg)
raise Exception(error_msg)
# Get expected file size from headers async with self.session.get(url, headers=self.headers, timeout=aiohttp.ClientTimeout(total=3600)) as response:
expected_size = int(response.headers.get('Content-Length', 0)) if response.status != 200:
if not expected_size: error_msg = f'Failed to download {tld}: {response.status} {await response.text()}'
logging.warning(f'No Content-Length header for {tld}') logging.error(error_msg)
if attempt + 1 < max_retries:
logging.info(f'Retrying {tld} in {retry_delay} seconds...')
await asyncio.sleep(retry_delay)
continue
raise Exception(error_msg)
if not (content_disposition := response.headers.get('Content-Disposition')): # Get expected file size from headers
error_msg = f'Missing Content-Disposition header for {tld}' expected_size = int(response.headers.get('Content-Length', 0))
logging.error(error_msg) if not expected_size:
raise ValueError(error_msg) logging.warning(f'No Content-Length header for {tld}')
filename = content_disposition.split('filename=')[-1].strip('"') if not (content_disposition := response.headers.get('Content-Disposition')):
filepath = os.path.join(output_directory, filename) error_msg = f'Missing Content-Disposition header for {tld}'
logging.error(error_msg)
raise ValueError(error_msg)
async with aiofiles.open(filepath, 'wb') as file: filename = content_disposition.split('filename=')[-1].strip('"')
total_size = 0 filepath = os.path.join(output_directory, filename)
last_progress = 0
async for chunk in response.content.iter_chunked(8192): async with aiofiles.open(filepath, 'wb') as file:
await file.write(chunk) total_size = 0
total_size += len(chunk) last_progress = 0
if expected_size: try:
progress = int((total_size / expected_size) * 100) async for chunk in response.content.iter_chunked(8192):
if progress >= last_progress + 5: # Log every 5% increase await file.write(chunk)
logging.info(f'Downloading {tld}: {progress}% ({total_size:,}/{expected_size:,} bytes)') total_size += len(chunk)
last_progress = progress if expected_size:
progress = int((total_size / expected_size) * 100)
if progress >= last_progress + 5:
logging.info(f'Downloading {tld}: {progress}% ({total_size:,}/{expected_size:,} bytes)')
last_progress = progress
except (asyncio.TimeoutError, aiohttp.ClientError) as e:
logging.error(f'Connection error while downloading {tld}: {str(e)}')
if attempt + 1 < max_retries:
logging.info(f'Retrying {tld} in {retry_delay} seconds...')
await asyncio.sleep(retry_delay)
continue
raise
# Verify file size # Verify file size
if expected_size and total_size != expected_size: if expected_size and total_size != expected_size:
error_msg = f'Incomplete download for {tld}: Got {total_size} bytes, expected {expected_size} bytes' error_msg = f'Incomplete download for {tld}: Got {total_size} bytes, expected {expected_size} bytes'
logging.error(error_msg) logging.error(error_msg)
os.remove(filepath) # Clean up incomplete file os.remove(filepath)
if attempt + 1 < max_retries:
logging.info(f'Retrying {tld} in {retry_delay} seconds...')
await asyncio.sleep(retry_delay)
continue
raise Exception(error_msg) raise Exception(error_msg)
size_mb = total_size / (1024 * 1024) size_mb = total_size / (1024 * 1024)
logging.info(f'Successfully downloaded {tld} zone file ({size_mb:.2f} MB)') logging.info(f'Successfully downloaded {tld} zone file ({size_mb:.2f} MB)')
if decompress: if decompress:
try: try:
# Verify gzip integrity before decompressing with gzip.open(filepath, 'rb') as test_gzip:
with gzip.open(filepath, 'rb') as test_gzip: test_gzip.read(1)
test_gzip.read(1) # Try reading first byte to verify gzip integrity
await self.gzip_decompress(filepath, cleanup) await self.gzip_decompress(filepath, cleanup)
filepath = filepath[:-3] # Remove .gz extension filepath = filepath[:-3]
logging.info(f'Decompressed {tld} zone file') logging.info(f'Decompressed {tld} zone file')
except (gzip.BadGzipFile, OSError) as e: except (gzip.BadGzipFile, OSError) as e:
error_msg = f'Failed to decompress {tld}: {str(e)}' error_msg = f'Failed to decompress {tld}: {str(e)}'
logging.error(error_msg) logging.error(error_msg)
os.remove(filepath) # Clean up corrupted file os.remove(filepath)
raise Exception(error_msg) raise Exception(error_msg)
return filepath return filepath
except Exception as e: except (aiohttp.ClientError, asyncio.TimeoutError) as e:
logging.error(f'Error downloading {tld}: {str(e)}') if attempt + 1 >= max_retries:
# Clean up any partial downloads logging.error(f'Failed to download {tld} after {max_retries} attempts: {str(e)}')
if 'filepath' in locals() and os.path.exists(filepath): if 'filepath' in locals() and os.path.exists(filepath):
os.remove(filepath) os.remove(filepath)
raise raise
logging.warning(f'Download attempt {attempt + 1} failed for {tld}: {str(e)}')
await asyncio.sleep(retry_delay)
except Exception as e:
logging.error(f'Error downloading {tld}: {str(e)}')
if 'filepath' in locals() and os.path.exists(filepath):
os.remove(filepath)
raise
if semaphore: if semaphore:
async with semaphore: async with semaphore:

2
setup.py
View File

@ -11,7 +11,7 @@ with open('README.md', 'r', encoding='utf-8') as fh:
setup( setup(
name='czds-api', name='czds-api',
version='1.2.8', version='1.2.9',
author='acidvegas', author='acidvegas',
author_email='acid.vegas@acid.vegas', author_email='acid.vegas@acid.vegas',
description='ICANN API for the Centralized Zones Data Service', description='ICANN API for the Centralized Zones Data Service',