changed image version; small typos
All checks were successful
Build Docker Image / build (push) Successful in 2m31s

This commit is contained in:
Tudor Sitaru
2025-10-23 17:00:25 +01:00
parent e8df11bc16
commit acfb22cbea
6 changed files with 149 additions and 116 deletions

View File

@@ -1,4 +1,4 @@
FROM python:3.11-slim FROM python:3.14-slim
# Set working directory # Set working directory
WORKDIR /app WORKDIR /app

View File

@@ -65,18 +65,6 @@ fi
run_with_logging "python3 src/config_snapshot_downloader.py --config $SNAPSHOT_CONFIG_FILE" "Config Snapshot Downloader" run_with_logging "python3 src/config_snapshot_downloader.py --config $SNAPSHOT_CONFIG_FILE" "Config Snapshot Downloader"
config_result=$? config_result=$?
# Run regular snapshot downloader with environment variables
if [ -n "$API_KEY" ]; then
run_with_logging "python3 src/snapshot_downloader.py --api-key $API_KEY --output-dir data/snapshots" "Snapshot Downloader (API Key)"
snapshot_result=$?
elif [ -n "$EMAIL" ] && [ -n "$PASSWORD" ]; then
run_with_logging "python3 src/snapshot_downloader.py --email $EMAIL --password $PASSWORD --output-dir data/snapshots" "Snapshot Downloader (Email/Password)"
snapshot_result=$?
else
log_message "WARNING: No authentication method provided via environment variables, skipping direct snapshot downloader"
snapshot_result=0
fi
# Summary # Summary
log_message "=== Daily Run Summary ===" log_message "=== Daily Run Summary ==="
if [ "$SKIP_ASSET_DOWNLOADER" = false ]; then if [ "$SKIP_ASSET_DOWNLOADER" = false ]; then

View File

@@ -37,22 +37,22 @@ class AssetTrackingTester:
"name": "family_photo_1.jpg", "name": "family_photo_1.jpg",
"updated": "2024-01-01T10:00:00Z", "updated": "2024-01-01T10:00:00Z",
"size": 1024000, "size": 1024000,
"mimeType": "image/jpeg" "mimeType": "image/jpeg",
}, },
{ {
"id": "asset_002", "id": "asset_002",
"name": "birthday_party.jpg", "name": "birthday_party.jpg",
"updated": "2024-01-02T15:30:00Z", "updated": "2024-01-02T15:30:00Z",
"size": 2048000, "size": 2048000,
"mimeType": "image/jpeg" "mimeType": "image/jpeg",
}, },
{ {
"id": "asset_003", "id": "asset_003",
"name": "school_event.png", "name": "school_event.png",
"updated": "2024-01-03T09:15:00Z", "updated": "2024-01-03T09:15:00Z",
"size": 1536000, "size": 1536000,
"mimeType": "image/png" "mimeType": "image/png",
} },
] ]
self.mock_assets_v2 = [ self.mock_assets_v2 = [
@@ -62,7 +62,7 @@ class AssetTrackingTester:
"name": "family_photo_1.jpg", "name": "family_photo_1.jpg",
"updated": "2024-01-01T10:00:00Z", "updated": "2024-01-01T10:00:00Z",
"size": 1024000, "size": 1024000,
"mimeType": "image/jpeg" "mimeType": "image/jpeg",
}, },
# Existing asset - modified # Existing asset - modified
{ {
@@ -70,7 +70,7 @@ class AssetTrackingTester:
"name": "birthday_party.jpg", "name": "birthday_party.jpg",
"updated": "2024-01-05T16:45:00Z", # Updated timestamp "updated": "2024-01-05T16:45:00Z", # Updated timestamp
"size": 2100000, # Different size "size": 2100000, # Different size
"mimeType": "image/jpeg" "mimeType": "image/jpeg",
}, },
# Existing asset - unchanged # Existing asset - unchanged
{ {
@@ -78,7 +78,7 @@ class AssetTrackingTester:
"name": "school_event.png", "name": "school_event.png",
"updated": "2024-01-03T09:15:00Z", "updated": "2024-01-03T09:15:00Z",
"size": 1536000, "size": 1536000,
"mimeType": "image/png" "mimeType": "image/png",
}, },
# New asset # New asset
{ {
@@ -86,8 +86,8 @@ class AssetTrackingTester:
"name": "new_vacation_photo.jpg", "name": "new_vacation_photo.jpg",
"updated": "2024-01-06T14:20:00Z", "updated": "2024-01-06T14:20:00Z",
"size": 3072000, "size": 3072000,
"mimeType": "image/jpeg" "mimeType": "image/jpeg",
} },
] ]
def test_basic_tracking(self): def test_basic_tracking(self):
@@ -111,7 +111,7 @@ class AssetTrackingTester:
# Simulate downloading first batch # Simulate downloading first batch
print("\n2. Simulating download of first batch...") print("\n2. Simulating download of first batch...")
for asset in self.mock_assets_v1: for asset in self.mock_assets_v1:
filename = asset['name'] filename = asset["name"]
filepath = Path(temp_dir) / filename filepath = Path(temp_dir) / filename
# Create dummy file # Create dummy file
@@ -149,7 +149,7 @@ class AssetTrackingTester:
# Simulate first batch download # Simulate first batch download
print("1. Simulating initial download...") print("1. Simulating initial download...")
for asset in self.mock_assets_v1: for asset in self.mock_assets_v1:
filename = asset['name'] filename = asset["name"]
filepath = Path(temp_dir) / filename filepath = Path(temp_dir) / filename
filepath.write_text(f"Mock content for {asset['id']}") filepath.write_text(f"Mock content for {asset['id']}")
tracker.mark_asset_downloaded(asset, filepath, True) tracker.mark_asset_downloaded(asset, filepath, True)
@@ -163,16 +163,22 @@ class AssetTrackingTester:
# Should detect 1 modified + 1 new = 2 assets # Should detect 1 modified + 1 new = 2 assets
expected = 2 # asset_002 (modified) + asset_004 (new) expected = 2 # asset_002 (modified) + asset_004 (new)
assert len(new_assets) == expected, f"Expected {expected} assets, got {len(new_assets)}" assert len(new_assets) == expected, (
f"Expected {expected} assets, got {len(new_assets)}"
)
# Check which assets were detected # Check which assets were detected
detected_ids = [asset['id'] for asset in new_assets] detected_ids = [asset["id"] for asset in new_assets]
print(f" Detected asset IDs: {detected_ids}") print(f" Detected asset IDs: {detected_ids}")
assert 'asset_002' in detected_ids, "Modified asset_002 should be detected" assert "asset_002" in detected_ids, "Modified asset_002 should be detected"
assert 'asset_004' in detected_ids, "New asset_004 should be detected" assert "asset_004" in detected_ids, "New asset_004 should be detected"
assert 'asset_001' not in detected_ids, "Unchanged asset_001 should not be detected" assert "asset_001" not in detected_ids, (
assert 'asset_003' not in detected_ids, "Unchanged asset_003 should not be detected" "Unchanged asset_001 should not be detected"
)
assert "asset_003" not in detected_ids, (
"Unchanged asset_003 should not be detected"
)
print(" ✅ Correctly identified 1 modified + 1 new asset") print(" ✅ Correctly identified 1 modified + 1 new asset")
print("✅ Modified asset detection test passed!") print("✅ Modified asset detection test passed!")
@@ -190,7 +196,7 @@ class AssetTrackingTester:
print("1. Creating and tracking assets...") print("1. Creating and tracking assets...")
filepaths = [] filepaths = []
for asset in self.mock_assets_v1: for asset in self.mock_assets_v1:
filename = asset['name'] filename = asset["name"]
filepath = Path(temp_dir) / filename filepath = Path(temp_dir) / filename
filepath.write_text(f"Mock content for {asset['id']}") filepath.write_text(f"Mock content for {asset['id']}")
tracker.mark_asset_downloaded(asset, filepath, True) tracker.mark_asset_downloaded(asset, filepath, True)
@@ -222,8 +228,12 @@ class AssetTrackingTester:
print(f" Missing files: {stats_after['missing_files']}") print(f" Missing files: {stats_after['missing_files']}")
# Verify cleanup worked # Verify cleanup worked
assert stats_after['missing_files'] == 0, "Should have no missing files after cleanup" assert stats_after["missing_files"] == 0, (
assert stats_after['total_tracked_assets'] == len(self.mock_assets_v1) - 1, "Should have one less tracked asset" "Should have no missing files after cleanup"
)
assert (
stats_after["total_tracked_assets"] == len(self.mock_assets_v1) - 1
), "Should have one less tracked asset"
print(" ✅ Cleanup successfully removed missing file metadata") print(" ✅ Cleanup successfully removed missing file metadata")
print("✅ Cleanup functionality test passed!") print("✅ Cleanup functionality test passed!")
@@ -246,7 +256,7 @@ class AssetTrackingTester:
list_endpoint="/v1/media/list", list_endpoint="/v1/media/list",
download_endpoint="/v1/media", download_endpoint="/v1/media",
output_dir=temp_dir, output_dir=temp_dir,
track_assets=True track_assets=True,
) )
# Check if asset tracker was initialized # Check if asset tracker was initialized
@@ -255,7 +265,9 @@ class AssetTrackingTester:
# Test tracker stats # Test tracker stats
stats = downloader.asset_tracker.get_stats() stats = downloader.asset_tracker.get_stats()
print(f" Initial stats: {stats['total_tracked_assets']} tracked assets") print(
f" Initial stats: {stats['total_tracked_assets']} tracked assets"
)
else: else:
print(" ❌ Asset tracker was not initialized") print(" ❌ Asset tracker was not initialized")
@@ -283,6 +295,7 @@ class AssetTrackingTester:
except Exception as e: except Exception as e:
print(f"\n❌ TEST FAILED: {e}") print(f"\n❌ TEST FAILED: {e}")
import traceback import traceback
traceback.print_exc() traceback.print_exc()
return False return False
@@ -310,7 +323,7 @@ async def test_with_real_api():
email=email, email=email,
password=password, password=password,
track_assets=True, track_assets=True,
max_concurrent=2 # Limit for testing max_concurrent=2, # Limit for testing
) )
print("\n1. First run - downloading all assets...") print("\n1. First run - downloading all assets...")
@@ -324,20 +337,23 @@ async def test_with_real_api():
print(f" Total size: {stats1['total_size_mb']} MB") print(f" Total size: {stats1['total_size_mb']} MB")
print("\n2. Second run - should find no new assets...") print("\n2. Second run - should find no new assets...")
downloader.stats = {'total': 0, 'successful': 0, 'failed': 0, 'skipped': 0} downloader.stats = {"total": 0, "successful": 0, "failed": 0, "skipped": 0}
await downloader.download_all_assets() await downloader.download_all_assets()
if downloader.asset_tracker: if downloader.asset_tracker:
stats2 = downloader.asset_tracker.get_stats() stats2 = downloader.asset_tracker.get_stats()
print(f"\nSecond run statistics:") print(f"\nSecond run statistics:")
print(f" New downloads: {downloader.stats['successful']}") print(f" New downloads: {downloader.stats['successful']}")
print(f" Skipped (unchanged): {len(stats2.get('total_tracked_assets', 0))}") print(
f" Skipped (unchanged): {len(stats2.get('total_tracked_assets', 0))}"
)
print("\n✅ Real API test completed!") print("\n✅ Real API test completed!")
except Exception as e: except Exception as e:
print(f"❌ Real API test failed: {e}") print(f"❌ Real API test failed: {e}")
import traceback import traceback
traceback.print_exc() traceback.print_exc()
@@ -346,7 +362,7 @@ def main():
# Setup logging # Setup logging
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
) )
tester = AssetTrackingTester() tester = AssetTrackingTester()
@@ -355,7 +371,7 @@ def main():
success = tester.run_all_tests() success = tester.run_all_tests()
# Ask user if they want to run real API test # Ask user if they want to run real API test
if success and len(sys.argv) > 1 and sys.argv[1] == '--real-api': if success and len(sys.argv) > 1 and sys.argv[1] == "--real-api":
print("\n" + "🌐 Running real API test...") print("\n" + "🌐 Running real API test...")
asyncio.run(test_with_real_api()) asyncio.run(test_with_real_api())

View File

@@ -39,7 +39,7 @@ class ConfigTrackingTester:
"timeout": 30, "timeout": 30,
"track_assets": track_assets, "track_assets": track_assets,
"email": "tudor.sitaru@gmail.com", "email": "tudor.sitaru@gmail.com",
"password": "mTVq8uNUvY7R39EPGVAm@" "password": "mTVq8uNUvY7R39EPGVAm@",
} }
def test_config_loading(self): def test_config_loading(self):
@@ -53,7 +53,7 @@ class ConfigTrackingTester:
# Test with tracking enabled # Test with tracking enabled
config_data = self.create_test_config(temp_dir, track_assets=True) config_data = self.create_test_config(temp_dir, track_assets=True)
with open(config_file, 'w') as f: with open(config_file, "w") as f:
json.dump(config_data, f, indent=2) json.dump(config_data, f, indent=2)
print("1. Testing config with asset tracking enabled...") print("1. Testing config with asset tracking enabled...")
@@ -67,7 +67,7 @@ class ConfigTrackingTester:
# Test with tracking disabled # Test with tracking disabled
config_data = self.create_test_config(temp_dir, track_assets=False) config_data = self.create_test_config(temp_dir, track_assets=False)
with open(config_file, 'w') as f: with open(config_file, "w") as f:
json.dump(config_data, f, indent=2) json.dump(config_data, f, indent=2)
print("\n2. Testing config with asset tracking disabled...") print("\n2. Testing config with asset tracking disabled...")
@@ -93,12 +93,14 @@ class ConfigTrackingTester:
# Create config without track_assets field # Create config without track_assets field
config_data = self.create_test_config(temp_dir) config_data = self.create_test_config(temp_dir)
del config_data['track_assets'] # Remove the field entirely del config_data["track_assets"] # Remove the field entirely
with open(config_file, 'w') as f: with open(config_file, "w") as f:
json.dump(config_data, f, indent=2) json.dump(config_data, f, indent=2)
print("1. Testing config without track_assets field (should default to True)...") print(
"1. Testing config without track_assets field (should default to True)..."
)
downloader = ConfigImageDownloader(str(config_file)) downloader = ConfigImageDownloader(str(config_file))
if downloader.asset_tracker: if downloader.asset_tracker:
@@ -121,7 +123,7 @@ class ConfigTrackingTester:
# Create config with tracking enabled # Create config with tracking enabled
config_data = self.create_test_config(temp_dir, track_assets=True) config_data = self.create_test_config(temp_dir, track_assets=True)
with open(config_file, 'w') as f: with open(config_file, "w") as f:
json.dump(config_data, f, indent=2) json.dump(config_data, f, indent=2)
print("1. Creating ConfigImageDownloader with tracking enabled...") print("1. Creating ConfigImageDownloader with tracking enabled...")
@@ -141,15 +143,15 @@ class ConfigTrackingTester:
"name": "test_image_1.jpg", "name": "test_image_1.jpg",
"updated": "2024-01-01T10:00:00Z", "updated": "2024-01-01T10:00:00Z",
"size": 1024000, "size": 1024000,
"mimeType": "image/jpeg" "mimeType": "image/jpeg",
}, },
{ {
"id": "config_test_002", "id": "config_test_002",
"name": "test_image_2.jpg", "name": "test_image_2.jpg",
"updated": "2024-01-02T11:00:00Z", "updated": "2024-01-02T11:00:00Z",
"size": 2048000, "size": 2048000,
"mimeType": "image/jpeg" "mimeType": "image/jpeg",
} },
] ]
# First check - should find all assets as new # First check - should find all assets as new
@@ -163,7 +165,7 @@ class ConfigTrackingTester:
# Simulate marking assets as downloaded # Simulate marking assets as downloaded
print("\n3. Simulating asset downloads...") print("\n3. Simulating asset downloads...")
for asset in mock_assets: for asset in mock_assets:
filepath = Path(temp_dir) / asset['name'] filepath = Path(temp_dir) / asset["name"]
filepath.write_text(f"Mock content for {asset['id']}") filepath.write_text(f"Mock content for {asset['id']}")
downloader.asset_tracker.mark_asset_downloaded(asset, filepath, True) downloader.asset_tracker.mark_asset_downloaded(asset, filepath, True)
print(f" Marked as downloaded: {asset['name']}") print(f" Marked as downloaded: {asset['name']}")
@@ -186,7 +188,7 @@ class ConfigTrackingTester:
print(f" Successful downloads: {stats['successful_downloads']}") print(f" Successful downloads: {stats['successful_downloads']}")
print(f" Existing files: {stats['existing_files']}") print(f" Existing files: {stats['existing_files']}")
if stats['total_tracked_assets'] != 2: if stats["total_tracked_assets"] != 2:
print(" ❌ Should have 2 tracked assets") print(" ❌ Should have 2 tracked assets")
return False return False
@@ -205,7 +207,7 @@ class ConfigTrackingTester:
# Create config with tracking enabled # Create config with tracking enabled
config_data = self.create_test_config(temp_dir, track_assets=True) config_data = self.create_test_config(temp_dir, track_assets=True)
with open(config_file, 'w') as f: with open(config_file, "w") as f:
json.dump(config_data, f, indent=2) json.dump(config_data, f, indent=2)
print("1. Testing --show-stats option...") print("1. Testing --show-stats option...")
@@ -218,17 +220,32 @@ class ConfigTrackingTester:
original_argv = sys.argv.copy() original_argv = sys.argv.copy()
# Test show-stats option # Test show-stats option
sys.argv = ['config_downloader.py', '--config', str(config_file), '--show-stats'] sys.argv = [
"config_downloader.py",
"--config",
str(config_file),
"--show-stats",
]
# This would normally call main(), but we'll just check the parsing works # This would normally call main(), but we'll just check the parsing works
print(" ✅ Command line parsing would work for --show-stats") print(" ✅ Command line parsing would work for --show-stats")
# Test cleanup option # Test cleanup option
sys.argv = ['config_downloader.py', '--config', str(config_file), '--cleanup'] sys.argv = [
"config_downloader.py",
"--config",
str(config_file),
"--cleanup",
]
print(" ✅ Command line parsing would work for --cleanup") print(" ✅ Command line parsing would work for --cleanup")
# Test force-redownload option # Test force-redownload option
sys.argv = ['config_downloader.py', '--config', str(config_file), '--force-redownload'] sys.argv = [
"config_downloader.py",
"--config",
str(config_file),
"--force-redownload",
]
print(" ✅ Command line parsing would work for --force-redownload") print(" ✅ Command line parsing would work for --force-redownload")
# Restore original argv # Restore original argv
@@ -258,8 +275,12 @@ class ConfigTrackingTester:
print("\n" + "=" * 80) print("\n" + "=" * 80)
print("🎉 ALL CONFIG DOWNLOADER TESTS PASSED!") print("🎉 ALL CONFIG DOWNLOADER TESTS PASSED!")
print("=" * 80) print("=" * 80)
print("✅ Asset tracking is now properly integrated into config_downloader.py") print(
print("✅ The config downloader will now skip already downloaded assets") "✅ Asset tracking is now properly integrated into config_downloader.py"
)
print(
"✅ The config downloader will now skip already downloaded assets"
)
print("✅ Command line options for tracking control are available") print("✅ Command line options for tracking control are available")
else: else:
print("\n❌ SOME TESTS FAILED") print("\n❌ SOME TESTS FAILED")
@@ -269,6 +290,7 @@ class ConfigTrackingTester:
except Exception as e: except Exception as e:
print(f"\n❌ TEST FAILED: {e}") print(f"\n❌ TEST FAILED: {e}")
import traceback import traceback
traceback.print_exc() traceback.print_exc()
return False return False
@@ -300,7 +322,9 @@ def show_usage_instructions():
print("python3 config_downloader.py --config parentzone_config.json") print("python3 config_downloader.py --config parentzone_config.json")
print() print()
print("# Force download all assets:") print("# Force download all assets:")
print("python3 config_downloader.py --config parentzone_config.json --force-redownload") print(
"python3 config_downloader.py --config parentzone_config.json --force-redownload"
)
print() print()
print("# Show asset statistics:") print("# Show asset statistics:")
print("python3 config_downloader.py --config parentzone_config.json --show-stats") print("python3 config_downloader.py --config parentzone_config.json --show-stats")
@@ -320,7 +344,7 @@ def main():
# Setup logging # Setup logging
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
) )
tester = ConfigTrackingTester() tester = ConfigTrackingTester()

View File

@@ -37,14 +37,18 @@ async def test_login():
print(f"User: {auth_manager.user_name}") print(f"User: {auth_manager.user_name}")
print(f"Provider: {auth_manager.provider_name}") print(f"Provider: {auth_manager.provider_name}")
print(f"User ID: {auth_manager.user_id}") print(f"User ID: {auth_manager.user_id}")
print(f"API Key: {auth_manager.api_key[:20]}..." if auth_manager.api_key else "No API key found") print(
f"API Key: {auth_manager.api_key[:20]}..."
if auth_manager.api_key
else "No API key found"
)
# Test getting auth headers # Test getting auth headers
headers = auth_manager.get_auth_headers() headers = auth_manager.get_auth_headers()
print(f"Auth headers: {list(headers.keys())}") print(f"Auth headers: {list(headers.keys())}")
if 'x-api-key' in headers: if "x-api-key" in headers:
print(f"✅ x-api-key header present: {headers['x-api-key'][:20]}...") print(f"✅ x-api-key header present: {headers['x-api-key'][:20]}...")
if 'x-api-product' in headers: if "x-api-product" in headers:
print(f"✅ x-api-product header: {headers['x-api-product']}") print(f"✅ x-api-product header: {headers['x-api-product']}")
# Test if authenticated # Test if authenticated

View File

@@ -44,7 +44,7 @@ class SnapshotDownloaderTester:
"date_to": "2024-01-31", # Small range for testing "date_to": "2024-01-31", # Small range for testing
"max_pages": 2, # Limit for testing "max_pages": 2, # Limit for testing
"email": self.email, "email": self.email,
"password": self.password "password": self.password,
} }
config.update(kwargs) config.update(kwargs)
return config return config
@@ -59,9 +59,7 @@ class SnapshotDownloaderTester:
print("1. Testing basic initialization...") print("1. Testing basic initialization...")
downloader = SnapshotDownloader( downloader = SnapshotDownloader(
output_dir=temp_dir, output_dir=temp_dir, email=self.email, password=self.password
email=self.email,
password=self.password
) )
# Check initialization # Check initialization
@@ -77,7 +75,7 @@ class SnapshotDownloaderTester:
print(" ❌ Email not set correctly") print(" ❌ Email not set correctly")
return False return False
if downloader.stats['total_snapshots'] == 0: if downloader.stats["total_snapshots"] == 0:
print(" ✅ Statistics initialized correctly") print(" ✅ Statistics initialized correctly")
else: else:
print(" ❌ Statistics not initialized correctly") print(" ❌ Statistics not initialized correctly")
@@ -85,8 +83,7 @@ class SnapshotDownloaderTester:
print("\n2. Testing with API key...") print("\n2. Testing with API key...")
downloader_api = SnapshotDownloader( downloader_api = SnapshotDownloader(
output_dir=temp_dir, output_dir=temp_dir, api_key=self.api_key
api_key=self.api_key
) )
if downloader_api.api_key == self.api_key: if downloader_api.api_key == self.api_key:
@@ -107,13 +104,10 @@ class SnapshotDownloaderTester:
with tempfile.TemporaryDirectory() as temp_dir: with tempfile.TemporaryDirectory() as temp_dir:
print("1. Testing API key headers...") print("1. Testing API key headers...")
downloader = SnapshotDownloader( downloader = SnapshotDownloader(output_dir=temp_dir, api_key=self.api_key)
output_dir=temp_dir,
api_key=self.api_key
)
headers = downloader.get_auth_headers() headers = downloader.get_auth_headers()
if 'x-api-key' in headers and headers['x-api-key'] == self.api_key: if "x-api-key" in headers and headers["x-api-key"] == self.api_key:
print(" ✅ API key header set correctly") print(" ✅ API key header set correctly")
else: else:
print(" ❌ API key header not set correctly") print(" ❌ API key header not set correctly")
@@ -121,8 +115,13 @@ class SnapshotDownloaderTester:
print("\n2. Testing standard headers...") print("\n2. Testing standard headers...")
expected_headers = [ expected_headers = [
'accept', 'accept-language', 'origin', 'user-agent', "accept",
'sec-fetch-dest', 'sec-fetch-mode', 'sec-fetch-site' "accept-language",
"origin",
"user-agent",
"sec-fetch-dest",
"sec-fetch-mode",
"sec-fetch-site",
] ]
for header in expected_headers: for header in expected_headers:
@@ -145,22 +144,23 @@ class SnapshotDownloaderTester:
print("1. Testing login authentication...") print("1. Testing login authentication...")
downloader = SnapshotDownloader( downloader = SnapshotDownloader(
output_dir=temp_dir, output_dir=temp_dir, email=self.email, password=self.password
email=self.email,
password=self.password
) )
try: try:
await downloader.authenticate() await downloader.authenticate()
if downloader.auth_manager and downloader.auth_manager.is_authenticated(): if (
downloader.auth_manager
and downloader.auth_manager.is_authenticated()
):
print(" ✅ Login authentication successful") print(" ✅ Login authentication successful")
# Check if API key was obtained # Check if API key was obtained
headers = downloader.get_auth_headers() headers = downloader.get_auth_headers()
if 'x-api-key' in headers: if "x-api-key" in headers:
print(" ✅ API key obtained from authentication") print(" ✅ API key obtained from authentication")
obtained_key = headers['x-api-key'] obtained_key = headers["x-api-key"]
if obtained_key: if obtained_key:
print(f" ✅ API key: {obtained_key[:20]}...") print(f" ✅ API key: {obtained_key[:20]}...")
else: else:
@@ -222,7 +222,7 @@ class SnapshotDownloaderTester:
date_from="2024-01-01", date_from="2024-01-01",
date_to="2024-01-31", date_to="2024-01-31",
page=1, page=1,
per_page=100 per_page=100,
) )
except Exception as e: except Exception as e:
# Expected - we just want to capture the URL # Expected - we just want to capture the URL
@@ -285,21 +285,12 @@ class SnapshotDownloaderTester:
"content": "This is a test snapshot with some content & special characters", "content": "This is a test snapshot with some content & special characters",
"created_at": "2024-01-15T10:30:00Z", "created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-15T10:30:00Z", "updated_at": "2024-01-15T10:30:00Z",
"author": { "author": {"name": "Test Author"},
"name": "Test Author" "child": {"name": "Test Child"},
}, "activity": {"name": "Test Activity"},
"child": {
"name": "Test Child"
},
"activity": {
"name": "Test Activity"
},
"images": [ "images": [
{ {"url": "https://example.com/image1.jpg", "name": "Test Image"}
"url": "https://example.com/image1.jpg", ],
"name": "Test Image"
}
]
} }
html = downloader.format_snapshot_html(mock_snapshot) html = downloader.format_snapshot_html(mock_snapshot)
@@ -389,7 +380,7 @@ class SnapshotDownloaderTester:
config_data = self.create_test_config(temp_dir) config_data = self.create_test_config(temp_dir)
config_file = Path(temp_dir) / "test_config.json" config_file = Path(temp_dir) / "test_config.json"
with open(config_file, 'w') as f: with open(config_file, "w") as f:
json.dump(config_data, f, indent=2) json.dump(config_data, f, indent=2)
# Test config loading # Test config loading
@@ -398,7 +389,7 @@ class SnapshotDownloaderTester:
print(" ✅ Configuration loaded successfully") print(" ✅ Configuration loaded successfully")
# Check if underlying downloader was created # Check if underlying downloader was created
if hasattr(config_downloader, 'downloader'): if hasattr(config_downloader, "downloader"):
print(" ✅ Underlying downloader created") print(" ✅ Underlying downloader created")
else: else:
print(" ❌ Underlying downloader not created") print(" ❌ Underlying downloader not created")
@@ -412,12 +403,12 @@ class SnapshotDownloaderTester:
# Test invalid config (missing auth) # Test invalid config (missing auth)
invalid_config = config_data.copy() invalid_config = config_data.copy()
del invalid_config['email'] del invalid_config["email"]
del invalid_config['password'] del invalid_config["password"]
# Don't set api_key either # Don't set api_key either
invalid_config_file = Path(temp_dir) / "invalid_config.json" invalid_config_file = Path(temp_dir) / "invalid_config.json"
with open(invalid_config_file, 'w') as f: with open(invalid_config_file, "w") as f:
json.dump(invalid_config, f, indent=2) json.dump(invalid_config, f, indent=2)
try: try:
@@ -449,7 +440,7 @@ class SnapshotDownloaderTester:
("2024-01-15T10:30:00.123Z", "2024-01-15 10:30:00"), ("2024-01-15T10:30:00.123Z", "2024-01-15 10:30:00"),
("2024-01-15T10:30:00+00:00", "2024-01-15 10:30:00"), ("2024-01-15T10:30:00+00:00", "2024-01-15 10:30:00"),
("invalid-date", "invalid-date"), # Should pass through unchanged ("invalid-date", "invalid-date"), # Should pass through unchanged
("", "") # Should handle empty string ("", ""), # Should handle empty string
] ]
for input_date, expected_prefix in test_dates: for input_date, expected_prefix in test_dates:
@@ -484,18 +475,18 @@ class SnapshotDownloaderTester:
# Page 1 # Page 1
{ {
"data": [{"id": "snap1"}, {"id": "snap2"}], "data": [{"id": "snap1"}, {"id": "snap2"}],
"pagination": {"current_page": 1, "last_page": 3} "pagination": {"current_page": 1, "last_page": 3},
}, },
# Page 2 # Page 2
{ {
"data": [{"id": "snap3"}, {"id": "snap4"}], "data": [{"id": "snap3"}, {"id": "snap4"}],
"pagination": {"current_page": 2, "last_page": 3} "pagination": {"current_page": 2, "last_page": 3},
}, },
# Page 3 # Page 3
{ {
"data": [{"id": "snap5"}], "data": [{"id": "snap5"}],
"pagination": {"current_page": 3, "last_page": 3} "pagination": {"current_page": 3, "last_page": 3},
} },
] ]
async def get(self, url, headers=None, timeout=None): async def get(self, url, headers=None, timeout=None):
@@ -523,10 +514,12 @@ class SnapshotDownloaderTester:
# Override the fetch_snapshots_page method to use our mock # Override the fetch_snapshots_page method to use our mock
original_method = downloader.fetch_snapshots_page original_method = downloader.fetch_snapshots_page
async def mock_fetch_page(session, type_ids, date_from, date_to, page, per_page): async def mock_fetch_page(
session, type_ids, date_from, date_to, page, per_page
):
response_data = mock_session.pages[page - 1] response_data = mock_session.pages[page - 1]
mock_session.call_count += 1 mock_session.call_count += 1
downloader.stats['pages_fetched'] += 1 downloader.stats["pages_fetched"] += 1
return response_data return response_data
downloader.fetch_snapshots_page = mock_fetch_page downloader.fetch_snapshots_page = mock_fetch_page
@@ -543,14 +536,16 @@ class SnapshotDownloaderTester:
print(f" ❌ Expected 5 snapshots, got {len(snapshots)}") print(f" ❌ Expected 5 snapshots, got {len(snapshots)}")
return False return False
if downloader.stats['pages_fetched'] == 3: if downloader.stats["pages_fetched"] == 3:
print(" ✅ Page count tracked correctly") print(" ✅ Page count tracked correctly")
else: else:
print(f" ❌ Expected 3 pages, tracked {downloader.stats['pages_fetched']}") print(
f" ❌ Expected 3 pages, tracked {downloader.stats['pages_fetched']}"
)
return False return False
# Test max_pages limit # Test max_pages limit
downloader.stats['pages_fetched'] = 0 # Reset downloader.stats["pages_fetched"] = 0 # Reset
mock_session.call_count = 0 # Reset mock_session.call_count = 0 # Reset
snapshots_limited = await downloader.fetch_all_snapshots( snapshots_limited = await downloader.fetch_all_snapshots(
@@ -560,7 +555,9 @@ class SnapshotDownloaderTester:
if len(snapshots_limited) == 4: # First 2 pages only if len(snapshots_limited) == 4: # First 2 pages only
print(" ✅ Max pages limit respected") print(" ✅ Max pages limit respected")
else: else:
print(f" ❌ Expected 4 snapshots with limit, got {len(snapshots_limited)}") print(
f" ❌ Expected 4 snapshots with limit, got {len(snapshots_limited)}"
)
return False return False
except Exception as e: except Exception as e:
@@ -622,7 +619,9 @@ def show_usage_examples():
print("python3 snapshot_downloader.py --email user@example.com --password password") print("python3 snapshot_downloader.py --email user@example.com --password password")
print() print()
print("# Specify date range") print("# Specify date range")
print("python3 snapshot_downloader.py --api-key KEY --date-from 2024-01-01 --date-to 2024-12-31") print(
"python3 snapshot_downloader.py --api-key KEY --date-from 2024-01-01 --date-to 2024-12-31"
)
print() print()
print("# Limit pages for testing") print("# Limit pages for testing")
print("python3 snapshot_downloader.py --api-key KEY --max-pages 5") print("python3 snapshot_downloader.py --api-key KEY --max-pages 5")
@@ -635,7 +634,9 @@ def show_usage_examples():
print("python3 config_snapshot_downloader.py --config snapshot_config.json") print("python3 config_snapshot_downloader.py --config snapshot_config.json")
print() print()
print("# Show config summary") print("# Show config summary")
print("python3 config_snapshot_downloader.py --config snapshot_config.json --show-config") print(
"python3 config_snapshot_downloader.py --config snapshot_config.json --show-config"
)
print("\n📄 Features:") print("\n📄 Features:")
print("• Downloads all snapshots with pagination support") print("• Downloads all snapshots with pagination support")