changed image version; small typos
All checks were successful
Build Docker Image / build (push) Successful in 2m31s
All checks were successful
Build Docker Image / build (push) Successful in 2m31s
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
FROM python:3.11-slim
|
||||
FROM python:3.14-slim
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
@@ -65,18 +65,6 @@ fi
|
||||
run_with_logging "python3 src/config_snapshot_downloader.py --config $SNAPSHOT_CONFIG_FILE" "Config Snapshot Downloader"
|
||||
config_result=$?
|
||||
|
||||
# Run regular snapshot downloader with environment variables
|
||||
if [ -n "$API_KEY" ]; then
|
||||
run_with_logging "python3 src/snapshot_downloader.py --api-key $API_KEY --output-dir data/snapshots" "Snapshot Downloader (API Key)"
|
||||
snapshot_result=$?
|
||||
elif [ -n "$EMAIL" ] && [ -n "$PASSWORD" ]; then
|
||||
run_with_logging "python3 src/snapshot_downloader.py --email $EMAIL --password $PASSWORD --output-dir data/snapshots" "Snapshot Downloader (Email/Password)"
|
||||
snapshot_result=$?
|
||||
else
|
||||
log_message "WARNING: No authentication method provided via environment variables, skipping direct snapshot downloader"
|
||||
snapshot_result=0
|
||||
fi
|
||||
|
||||
# Summary
|
||||
log_message "=== Daily Run Summary ==="
|
||||
if [ "$SKIP_ASSET_DOWNLOADER" = false ]; then
|
||||
|
||||
@@ -37,22 +37,22 @@ class AssetTrackingTester:
|
||||
"name": "family_photo_1.jpg",
|
||||
"updated": "2024-01-01T10:00:00Z",
|
||||
"size": 1024000,
|
||||
"mimeType": "image/jpeg"
|
||||
"mimeType": "image/jpeg",
|
||||
},
|
||||
{
|
||||
"id": "asset_002",
|
||||
"name": "birthday_party.jpg",
|
||||
"updated": "2024-01-02T15:30:00Z",
|
||||
"size": 2048000,
|
||||
"mimeType": "image/jpeg"
|
||||
"mimeType": "image/jpeg",
|
||||
},
|
||||
{
|
||||
"id": "asset_003",
|
||||
"name": "school_event.png",
|
||||
"updated": "2024-01-03T09:15:00Z",
|
||||
"size": 1536000,
|
||||
"mimeType": "image/png"
|
||||
}
|
||||
"mimeType": "image/png",
|
||||
},
|
||||
]
|
||||
|
||||
self.mock_assets_v2 = [
|
||||
@@ -62,7 +62,7 @@ class AssetTrackingTester:
|
||||
"name": "family_photo_1.jpg",
|
||||
"updated": "2024-01-01T10:00:00Z",
|
||||
"size": 1024000,
|
||||
"mimeType": "image/jpeg"
|
||||
"mimeType": "image/jpeg",
|
||||
},
|
||||
# Existing asset - modified
|
||||
{
|
||||
@@ -70,7 +70,7 @@ class AssetTrackingTester:
|
||||
"name": "birthday_party.jpg",
|
||||
"updated": "2024-01-05T16:45:00Z", # Updated timestamp
|
||||
"size": 2100000, # Different size
|
||||
"mimeType": "image/jpeg"
|
||||
"mimeType": "image/jpeg",
|
||||
},
|
||||
# Existing asset - unchanged
|
||||
{
|
||||
@@ -78,7 +78,7 @@ class AssetTrackingTester:
|
||||
"name": "school_event.png",
|
||||
"updated": "2024-01-03T09:15:00Z",
|
||||
"size": 1536000,
|
||||
"mimeType": "image/png"
|
||||
"mimeType": "image/png",
|
||||
},
|
||||
# New asset
|
||||
{
|
||||
@@ -86,8 +86,8 @@ class AssetTrackingTester:
|
||||
"name": "new_vacation_photo.jpg",
|
||||
"updated": "2024-01-06T14:20:00Z",
|
||||
"size": 3072000,
|
||||
"mimeType": "image/jpeg"
|
||||
}
|
||||
"mimeType": "image/jpeg",
|
||||
},
|
||||
]
|
||||
|
||||
def test_basic_tracking(self):
|
||||
@@ -111,7 +111,7 @@ class AssetTrackingTester:
|
||||
# Simulate downloading first batch
|
||||
print("\n2. Simulating download of first batch...")
|
||||
for asset in self.mock_assets_v1:
|
||||
filename = asset['name']
|
||||
filename = asset["name"]
|
||||
filepath = Path(temp_dir) / filename
|
||||
|
||||
# Create dummy file
|
||||
@@ -149,7 +149,7 @@ class AssetTrackingTester:
|
||||
# Simulate first batch download
|
||||
print("1. Simulating initial download...")
|
||||
for asset in self.mock_assets_v1:
|
||||
filename = asset['name']
|
||||
filename = asset["name"]
|
||||
filepath = Path(temp_dir) / filename
|
||||
filepath.write_text(f"Mock content for {asset['id']}")
|
||||
tracker.mark_asset_downloaded(asset, filepath, True)
|
||||
@@ -163,16 +163,22 @@ class AssetTrackingTester:
|
||||
|
||||
# Should detect 1 modified + 1 new = 2 assets
|
||||
expected = 2 # asset_002 (modified) + asset_004 (new)
|
||||
assert len(new_assets) == expected, f"Expected {expected} assets, got {len(new_assets)}"
|
||||
assert len(new_assets) == expected, (
|
||||
f"Expected {expected} assets, got {len(new_assets)}"
|
||||
)
|
||||
|
||||
# Check which assets were detected
|
||||
detected_ids = [asset['id'] for asset in new_assets]
|
||||
detected_ids = [asset["id"] for asset in new_assets]
|
||||
print(f" Detected asset IDs: {detected_ids}")
|
||||
|
||||
assert 'asset_002' in detected_ids, "Modified asset_002 should be detected"
|
||||
assert 'asset_004' in detected_ids, "New asset_004 should be detected"
|
||||
assert 'asset_001' not in detected_ids, "Unchanged asset_001 should not be detected"
|
||||
assert 'asset_003' not in detected_ids, "Unchanged asset_003 should not be detected"
|
||||
assert "asset_002" in detected_ids, "Modified asset_002 should be detected"
|
||||
assert "asset_004" in detected_ids, "New asset_004 should be detected"
|
||||
assert "asset_001" not in detected_ids, (
|
||||
"Unchanged asset_001 should not be detected"
|
||||
)
|
||||
assert "asset_003" not in detected_ids, (
|
||||
"Unchanged asset_003 should not be detected"
|
||||
)
|
||||
|
||||
print(" ✅ Correctly identified 1 modified + 1 new asset")
|
||||
print("✅ Modified asset detection test passed!")
|
||||
@@ -190,7 +196,7 @@ class AssetTrackingTester:
|
||||
print("1. Creating and tracking assets...")
|
||||
filepaths = []
|
||||
for asset in self.mock_assets_v1:
|
||||
filename = asset['name']
|
||||
filename = asset["name"]
|
||||
filepath = Path(temp_dir) / filename
|
||||
filepath.write_text(f"Mock content for {asset['id']}")
|
||||
tracker.mark_asset_downloaded(asset, filepath, True)
|
||||
@@ -222,8 +228,12 @@ class AssetTrackingTester:
|
||||
print(f" Missing files: {stats_after['missing_files']}")
|
||||
|
||||
# Verify cleanup worked
|
||||
assert stats_after['missing_files'] == 0, "Should have no missing files after cleanup"
|
||||
assert stats_after['total_tracked_assets'] == len(self.mock_assets_v1) - 1, "Should have one less tracked asset"
|
||||
assert stats_after["missing_files"] == 0, (
|
||||
"Should have no missing files after cleanup"
|
||||
)
|
||||
assert (
|
||||
stats_after["total_tracked_assets"] == len(self.mock_assets_v1) - 1
|
||||
), "Should have one less tracked asset"
|
||||
|
||||
print(" ✅ Cleanup successfully removed missing file metadata")
|
||||
print("✅ Cleanup functionality test passed!")
|
||||
@@ -246,7 +256,7 @@ class AssetTrackingTester:
|
||||
list_endpoint="/v1/media/list",
|
||||
download_endpoint="/v1/media",
|
||||
output_dir=temp_dir,
|
||||
track_assets=True
|
||||
track_assets=True,
|
||||
)
|
||||
|
||||
# Check if asset tracker was initialized
|
||||
@@ -255,7 +265,9 @@ class AssetTrackingTester:
|
||||
|
||||
# Test tracker stats
|
||||
stats = downloader.asset_tracker.get_stats()
|
||||
print(f" Initial stats: {stats['total_tracked_assets']} tracked assets")
|
||||
print(
|
||||
f" Initial stats: {stats['total_tracked_assets']} tracked assets"
|
||||
)
|
||||
else:
|
||||
print(" ❌ Asset tracker was not initialized")
|
||||
|
||||
@@ -283,6 +295,7 @@ class AssetTrackingTester:
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST FAILED: {e}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
@@ -310,7 +323,7 @@ async def test_with_real_api():
|
||||
email=email,
|
||||
password=password,
|
||||
track_assets=True,
|
||||
max_concurrent=2 # Limit for testing
|
||||
max_concurrent=2, # Limit for testing
|
||||
)
|
||||
|
||||
print("\n1. First run - downloading all assets...")
|
||||
@@ -324,20 +337,23 @@ async def test_with_real_api():
|
||||
print(f" Total size: {stats1['total_size_mb']} MB")
|
||||
|
||||
print("\n2. Second run - should find no new assets...")
|
||||
downloader.stats = {'total': 0, 'successful': 0, 'failed': 0, 'skipped': 0}
|
||||
downloader.stats = {"total": 0, "successful": 0, "failed": 0, "skipped": 0}
|
||||
await downloader.download_all_assets()
|
||||
|
||||
if downloader.asset_tracker:
|
||||
stats2 = downloader.asset_tracker.get_stats()
|
||||
print(f"\nSecond run statistics:")
|
||||
print(f" New downloads: {downloader.stats['successful']}")
|
||||
print(f" Skipped (unchanged): {len(stats2.get('total_tracked_assets', 0))}")
|
||||
print(
|
||||
f" Skipped (unchanged): {len(stats2.get('total_tracked_assets', 0))}"
|
||||
)
|
||||
|
||||
print("\n✅ Real API test completed!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Real API test failed: {e}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
@@ -346,7 +362,7 @@ def main():
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
|
||||
tester = AssetTrackingTester()
|
||||
@@ -355,7 +371,7 @@ def main():
|
||||
success = tester.run_all_tests()
|
||||
|
||||
# Ask user if they want to run real API test
|
||||
if success and len(sys.argv) > 1 and sys.argv[1] == '--real-api':
|
||||
if success and len(sys.argv) > 1 and sys.argv[1] == "--real-api":
|
||||
print("\n" + "🌐 Running real API test...")
|
||||
asyncio.run(test_with_real_api())
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ class ConfigTrackingTester:
|
||||
"timeout": 30,
|
||||
"track_assets": track_assets,
|
||||
"email": "tudor.sitaru@gmail.com",
|
||||
"password": "mTVq8uNUvY7R39EPGVAm@"
|
||||
"password": "mTVq8uNUvY7R39EPGVAm@",
|
||||
}
|
||||
|
||||
def test_config_loading(self):
|
||||
@@ -53,7 +53,7 @@ class ConfigTrackingTester:
|
||||
|
||||
# Test with tracking enabled
|
||||
config_data = self.create_test_config(temp_dir, track_assets=True)
|
||||
with open(config_file, 'w') as f:
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(config_data, f, indent=2)
|
||||
|
||||
print("1. Testing config with asset tracking enabled...")
|
||||
@@ -67,7 +67,7 @@ class ConfigTrackingTester:
|
||||
|
||||
# Test with tracking disabled
|
||||
config_data = self.create_test_config(temp_dir, track_assets=False)
|
||||
with open(config_file, 'w') as f:
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(config_data, f, indent=2)
|
||||
|
||||
print("\n2. Testing config with asset tracking disabled...")
|
||||
@@ -93,12 +93,14 @@ class ConfigTrackingTester:
|
||||
|
||||
# Create config without track_assets field
|
||||
config_data = self.create_test_config(temp_dir)
|
||||
del config_data['track_assets'] # Remove the field entirely
|
||||
del config_data["track_assets"] # Remove the field entirely
|
||||
|
||||
with open(config_file, 'w') as f:
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(config_data, f, indent=2)
|
||||
|
||||
print("1. Testing config without track_assets field (should default to True)...")
|
||||
print(
|
||||
"1. Testing config without track_assets field (should default to True)..."
|
||||
)
|
||||
downloader = ConfigImageDownloader(str(config_file))
|
||||
|
||||
if downloader.asset_tracker:
|
||||
@@ -121,7 +123,7 @@ class ConfigTrackingTester:
|
||||
|
||||
# Create config with tracking enabled
|
||||
config_data = self.create_test_config(temp_dir, track_assets=True)
|
||||
with open(config_file, 'w') as f:
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(config_data, f, indent=2)
|
||||
|
||||
print("1. Creating ConfigImageDownloader with tracking enabled...")
|
||||
@@ -141,15 +143,15 @@ class ConfigTrackingTester:
|
||||
"name": "test_image_1.jpg",
|
||||
"updated": "2024-01-01T10:00:00Z",
|
||||
"size": 1024000,
|
||||
"mimeType": "image/jpeg"
|
||||
"mimeType": "image/jpeg",
|
||||
},
|
||||
{
|
||||
"id": "config_test_002",
|
||||
"name": "test_image_2.jpg",
|
||||
"updated": "2024-01-02T11:00:00Z",
|
||||
"size": 2048000,
|
||||
"mimeType": "image/jpeg"
|
||||
}
|
||||
"mimeType": "image/jpeg",
|
||||
},
|
||||
]
|
||||
|
||||
# First check - should find all assets as new
|
||||
@@ -163,7 +165,7 @@ class ConfigTrackingTester:
|
||||
# Simulate marking assets as downloaded
|
||||
print("\n3. Simulating asset downloads...")
|
||||
for asset in mock_assets:
|
||||
filepath = Path(temp_dir) / asset['name']
|
||||
filepath = Path(temp_dir) / asset["name"]
|
||||
filepath.write_text(f"Mock content for {asset['id']}")
|
||||
downloader.asset_tracker.mark_asset_downloaded(asset, filepath, True)
|
||||
print(f" Marked as downloaded: {asset['name']}")
|
||||
@@ -186,7 +188,7 @@ class ConfigTrackingTester:
|
||||
print(f" Successful downloads: {stats['successful_downloads']}")
|
||||
print(f" Existing files: {stats['existing_files']}")
|
||||
|
||||
if stats['total_tracked_assets'] != 2:
|
||||
if stats["total_tracked_assets"] != 2:
|
||||
print(" ❌ Should have 2 tracked assets")
|
||||
return False
|
||||
|
||||
@@ -205,7 +207,7 @@ class ConfigTrackingTester:
|
||||
|
||||
# Create config with tracking enabled
|
||||
config_data = self.create_test_config(temp_dir, track_assets=True)
|
||||
with open(config_file, 'w') as f:
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(config_data, f, indent=2)
|
||||
|
||||
print("1. Testing --show-stats option...")
|
||||
@@ -218,17 +220,32 @@ class ConfigTrackingTester:
|
||||
original_argv = sys.argv.copy()
|
||||
|
||||
# Test show-stats option
|
||||
sys.argv = ['config_downloader.py', '--config', str(config_file), '--show-stats']
|
||||
sys.argv = [
|
||||
"config_downloader.py",
|
||||
"--config",
|
||||
str(config_file),
|
||||
"--show-stats",
|
||||
]
|
||||
|
||||
# This would normally call main(), but we'll just check the parsing works
|
||||
print(" ✅ Command line parsing would work for --show-stats")
|
||||
|
||||
# Test cleanup option
|
||||
sys.argv = ['config_downloader.py', '--config', str(config_file), '--cleanup']
|
||||
sys.argv = [
|
||||
"config_downloader.py",
|
||||
"--config",
|
||||
str(config_file),
|
||||
"--cleanup",
|
||||
]
|
||||
print(" ✅ Command line parsing would work for --cleanup")
|
||||
|
||||
# Test force-redownload option
|
||||
sys.argv = ['config_downloader.py', '--config', str(config_file), '--force-redownload']
|
||||
sys.argv = [
|
||||
"config_downloader.py",
|
||||
"--config",
|
||||
str(config_file),
|
||||
"--force-redownload",
|
||||
]
|
||||
print(" ✅ Command line parsing would work for --force-redownload")
|
||||
|
||||
# Restore original argv
|
||||
@@ -258,8 +275,12 @@ class ConfigTrackingTester:
|
||||
print("\n" + "=" * 80)
|
||||
print("🎉 ALL CONFIG DOWNLOADER TESTS PASSED!")
|
||||
print("=" * 80)
|
||||
print("✅ Asset tracking is now properly integrated into config_downloader.py")
|
||||
print("✅ The config downloader will now skip already downloaded assets")
|
||||
print(
|
||||
"✅ Asset tracking is now properly integrated into config_downloader.py"
|
||||
)
|
||||
print(
|
||||
"✅ The config downloader will now skip already downloaded assets"
|
||||
)
|
||||
print("✅ Command line options for tracking control are available")
|
||||
else:
|
||||
print("\n❌ SOME TESTS FAILED")
|
||||
@@ -269,6 +290,7 @@ class ConfigTrackingTester:
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST FAILED: {e}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
@@ -300,7 +322,9 @@ def show_usage_instructions():
|
||||
print("python3 config_downloader.py --config parentzone_config.json")
|
||||
print()
|
||||
print("# Force download all assets:")
|
||||
print("python3 config_downloader.py --config parentzone_config.json --force-redownload")
|
||||
print(
|
||||
"python3 config_downloader.py --config parentzone_config.json --force-redownload"
|
||||
)
|
||||
print()
|
||||
print("# Show asset statistics:")
|
||||
print("python3 config_downloader.py --config parentzone_config.json --show-stats")
|
||||
@@ -320,7 +344,7 @@ def main():
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
|
||||
tester = ConfigTrackingTester()
|
||||
|
||||
@@ -37,14 +37,18 @@ async def test_login():
|
||||
print(f"User: {auth_manager.user_name}")
|
||||
print(f"Provider: {auth_manager.provider_name}")
|
||||
print(f"User ID: {auth_manager.user_id}")
|
||||
print(f"API Key: {auth_manager.api_key[:20]}..." if auth_manager.api_key else "No API key found")
|
||||
print(
|
||||
f"API Key: {auth_manager.api_key[:20]}..."
|
||||
if auth_manager.api_key
|
||||
else "No API key found"
|
||||
)
|
||||
|
||||
# Test getting auth headers
|
||||
headers = auth_manager.get_auth_headers()
|
||||
print(f"Auth headers: {list(headers.keys())}")
|
||||
if 'x-api-key' in headers:
|
||||
if "x-api-key" in headers:
|
||||
print(f"✅ x-api-key header present: {headers['x-api-key'][:20]}...")
|
||||
if 'x-api-product' in headers:
|
||||
if "x-api-product" in headers:
|
||||
print(f"✅ x-api-product header: {headers['x-api-product']}")
|
||||
|
||||
# Test if authenticated
|
||||
|
||||
@@ -44,7 +44,7 @@ class SnapshotDownloaderTester:
|
||||
"date_to": "2024-01-31", # Small range for testing
|
||||
"max_pages": 2, # Limit for testing
|
||||
"email": self.email,
|
||||
"password": self.password
|
||||
"password": self.password,
|
||||
}
|
||||
config.update(kwargs)
|
||||
return config
|
||||
@@ -59,9 +59,7 @@ class SnapshotDownloaderTester:
|
||||
print("1. Testing basic initialization...")
|
||||
|
||||
downloader = SnapshotDownloader(
|
||||
output_dir=temp_dir,
|
||||
email=self.email,
|
||||
password=self.password
|
||||
output_dir=temp_dir, email=self.email, password=self.password
|
||||
)
|
||||
|
||||
# Check initialization
|
||||
@@ -77,7 +75,7 @@ class SnapshotDownloaderTester:
|
||||
print(" ❌ Email not set correctly")
|
||||
return False
|
||||
|
||||
if downloader.stats['total_snapshots'] == 0:
|
||||
if downloader.stats["total_snapshots"] == 0:
|
||||
print(" ✅ Statistics initialized correctly")
|
||||
else:
|
||||
print(" ❌ Statistics not initialized correctly")
|
||||
@@ -85,8 +83,7 @@ class SnapshotDownloaderTester:
|
||||
|
||||
print("\n2. Testing with API key...")
|
||||
downloader_api = SnapshotDownloader(
|
||||
output_dir=temp_dir,
|
||||
api_key=self.api_key
|
||||
output_dir=temp_dir, api_key=self.api_key
|
||||
)
|
||||
|
||||
if downloader_api.api_key == self.api_key:
|
||||
@@ -107,13 +104,10 @@ class SnapshotDownloaderTester:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
print("1. Testing API key headers...")
|
||||
|
||||
downloader = SnapshotDownloader(
|
||||
output_dir=temp_dir,
|
||||
api_key=self.api_key
|
||||
)
|
||||
downloader = SnapshotDownloader(output_dir=temp_dir, api_key=self.api_key)
|
||||
|
||||
headers = downloader.get_auth_headers()
|
||||
if 'x-api-key' in headers and headers['x-api-key'] == self.api_key:
|
||||
if "x-api-key" in headers and headers["x-api-key"] == self.api_key:
|
||||
print(" ✅ API key header set correctly")
|
||||
else:
|
||||
print(" ❌ API key header not set correctly")
|
||||
@@ -121,8 +115,13 @@ class SnapshotDownloaderTester:
|
||||
|
||||
print("\n2. Testing standard headers...")
|
||||
expected_headers = [
|
||||
'accept', 'accept-language', 'origin', 'user-agent',
|
||||
'sec-fetch-dest', 'sec-fetch-mode', 'sec-fetch-site'
|
||||
"accept",
|
||||
"accept-language",
|
||||
"origin",
|
||||
"user-agent",
|
||||
"sec-fetch-dest",
|
||||
"sec-fetch-mode",
|
||||
"sec-fetch-site",
|
||||
]
|
||||
|
||||
for header in expected_headers:
|
||||
@@ -145,22 +144,23 @@ class SnapshotDownloaderTester:
|
||||
print("1. Testing login authentication...")
|
||||
|
||||
downloader = SnapshotDownloader(
|
||||
output_dir=temp_dir,
|
||||
email=self.email,
|
||||
password=self.password
|
||||
output_dir=temp_dir, email=self.email, password=self.password
|
||||
)
|
||||
|
||||
try:
|
||||
await downloader.authenticate()
|
||||
|
||||
if downloader.auth_manager and downloader.auth_manager.is_authenticated():
|
||||
if (
|
||||
downloader.auth_manager
|
||||
and downloader.auth_manager.is_authenticated()
|
||||
):
|
||||
print(" ✅ Login authentication successful")
|
||||
|
||||
# Check if API key was obtained
|
||||
headers = downloader.get_auth_headers()
|
||||
if 'x-api-key' in headers:
|
||||
if "x-api-key" in headers:
|
||||
print(" ✅ API key obtained from authentication")
|
||||
obtained_key = headers['x-api-key']
|
||||
obtained_key = headers["x-api-key"]
|
||||
if obtained_key:
|
||||
print(f" ✅ API key: {obtained_key[:20]}...")
|
||||
else:
|
||||
@@ -222,7 +222,7 @@ class SnapshotDownloaderTester:
|
||||
date_from="2024-01-01",
|
||||
date_to="2024-01-31",
|
||||
page=1,
|
||||
per_page=100
|
||||
per_page=100,
|
||||
)
|
||||
except Exception as e:
|
||||
# Expected - we just want to capture the URL
|
||||
@@ -285,21 +285,12 @@ class SnapshotDownloaderTester:
|
||||
"content": "This is a test snapshot with some content & special characters",
|
||||
"created_at": "2024-01-15T10:30:00Z",
|
||||
"updated_at": "2024-01-15T10:30:00Z",
|
||||
"author": {
|
||||
"name": "Test Author"
|
||||
},
|
||||
"child": {
|
||||
"name": "Test Child"
|
||||
},
|
||||
"activity": {
|
||||
"name": "Test Activity"
|
||||
},
|
||||
"author": {"name": "Test Author"},
|
||||
"child": {"name": "Test Child"},
|
||||
"activity": {"name": "Test Activity"},
|
||||
"images": [
|
||||
{
|
||||
"url": "https://example.com/image1.jpg",
|
||||
"name": "Test Image"
|
||||
}
|
||||
]
|
||||
{"url": "https://example.com/image1.jpg", "name": "Test Image"}
|
||||
],
|
||||
}
|
||||
|
||||
html = downloader.format_snapshot_html(mock_snapshot)
|
||||
@@ -389,7 +380,7 @@ class SnapshotDownloaderTester:
|
||||
config_data = self.create_test_config(temp_dir)
|
||||
config_file = Path(temp_dir) / "test_config.json"
|
||||
|
||||
with open(config_file, 'w') as f:
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(config_data, f, indent=2)
|
||||
|
||||
# Test config loading
|
||||
@@ -398,7 +389,7 @@ class SnapshotDownloaderTester:
|
||||
print(" ✅ Configuration loaded successfully")
|
||||
|
||||
# Check if underlying downloader was created
|
||||
if hasattr(config_downloader, 'downloader'):
|
||||
if hasattr(config_downloader, "downloader"):
|
||||
print(" ✅ Underlying downloader created")
|
||||
else:
|
||||
print(" ❌ Underlying downloader not created")
|
||||
@@ -412,12 +403,12 @@ class SnapshotDownloaderTester:
|
||||
|
||||
# Test invalid config (missing auth)
|
||||
invalid_config = config_data.copy()
|
||||
del invalid_config['email']
|
||||
del invalid_config['password']
|
||||
del invalid_config["email"]
|
||||
del invalid_config["password"]
|
||||
# Don't set api_key either
|
||||
|
||||
invalid_config_file = Path(temp_dir) / "invalid_config.json"
|
||||
with open(invalid_config_file, 'w') as f:
|
||||
with open(invalid_config_file, "w") as f:
|
||||
json.dump(invalid_config, f, indent=2)
|
||||
|
||||
try:
|
||||
@@ -449,7 +440,7 @@ class SnapshotDownloaderTester:
|
||||
("2024-01-15T10:30:00.123Z", "2024-01-15 10:30:00"),
|
||||
("2024-01-15T10:30:00+00:00", "2024-01-15 10:30:00"),
|
||||
("invalid-date", "invalid-date"), # Should pass through unchanged
|
||||
("", "") # Should handle empty string
|
||||
("", ""), # Should handle empty string
|
||||
]
|
||||
|
||||
for input_date, expected_prefix in test_dates:
|
||||
@@ -484,18 +475,18 @@ class SnapshotDownloaderTester:
|
||||
# Page 1
|
||||
{
|
||||
"data": [{"id": "snap1"}, {"id": "snap2"}],
|
||||
"pagination": {"current_page": 1, "last_page": 3}
|
||||
"pagination": {"current_page": 1, "last_page": 3},
|
||||
},
|
||||
# Page 2
|
||||
{
|
||||
"data": [{"id": "snap3"}, {"id": "snap4"}],
|
||||
"pagination": {"current_page": 2, "last_page": 3}
|
||||
"pagination": {"current_page": 2, "last_page": 3},
|
||||
},
|
||||
# Page 3
|
||||
{
|
||||
"data": [{"id": "snap5"}],
|
||||
"pagination": {"current_page": 3, "last_page": 3}
|
||||
}
|
||||
"pagination": {"current_page": 3, "last_page": 3},
|
||||
},
|
||||
]
|
||||
|
||||
async def get(self, url, headers=None, timeout=None):
|
||||
@@ -523,10 +514,12 @@ class SnapshotDownloaderTester:
|
||||
# Override the fetch_snapshots_page method to use our mock
|
||||
original_method = downloader.fetch_snapshots_page
|
||||
|
||||
async def mock_fetch_page(session, type_ids, date_from, date_to, page, per_page):
|
||||
async def mock_fetch_page(
|
||||
session, type_ids, date_from, date_to, page, per_page
|
||||
):
|
||||
response_data = mock_session.pages[page - 1]
|
||||
mock_session.call_count += 1
|
||||
downloader.stats['pages_fetched'] += 1
|
||||
downloader.stats["pages_fetched"] += 1
|
||||
return response_data
|
||||
|
||||
downloader.fetch_snapshots_page = mock_fetch_page
|
||||
@@ -543,14 +536,16 @@ class SnapshotDownloaderTester:
|
||||
print(f" ❌ Expected 5 snapshots, got {len(snapshots)}")
|
||||
return False
|
||||
|
||||
if downloader.stats['pages_fetched'] == 3:
|
||||
if downloader.stats["pages_fetched"] == 3:
|
||||
print(" ✅ Page count tracked correctly")
|
||||
else:
|
||||
print(f" ❌ Expected 3 pages, tracked {downloader.stats['pages_fetched']}")
|
||||
print(
|
||||
f" ❌ Expected 3 pages, tracked {downloader.stats['pages_fetched']}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Test max_pages limit
|
||||
downloader.stats['pages_fetched'] = 0 # Reset
|
||||
downloader.stats["pages_fetched"] = 0 # Reset
|
||||
mock_session.call_count = 0 # Reset
|
||||
|
||||
snapshots_limited = await downloader.fetch_all_snapshots(
|
||||
@@ -560,7 +555,9 @@ class SnapshotDownloaderTester:
|
||||
if len(snapshots_limited) == 4: # First 2 pages only
|
||||
print(" ✅ Max pages limit respected")
|
||||
else:
|
||||
print(f" ❌ Expected 4 snapshots with limit, got {len(snapshots_limited)}")
|
||||
print(
|
||||
f" ❌ Expected 4 snapshots with limit, got {len(snapshots_limited)}"
|
||||
)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
@@ -622,7 +619,9 @@ def show_usage_examples():
|
||||
print("python3 snapshot_downloader.py --email user@example.com --password password")
|
||||
print()
|
||||
print("# Specify date range")
|
||||
print("python3 snapshot_downloader.py --api-key KEY --date-from 2024-01-01 --date-to 2024-12-31")
|
||||
print(
|
||||
"python3 snapshot_downloader.py --api-key KEY --date-from 2024-01-01 --date-to 2024-12-31"
|
||||
)
|
||||
print()
|
||||
print("# Limit pages for testing")
|
||||
print("python3 snapshot_downloader.py --api-key KEY --max-pages 5")
|
||||
@@ -635,7 +634,9 @@ def show_usage_examples():
|
||||
print("python3 config_snapshot_downloader.py --config snapshot_config.json")
|
||||
print()
|
||||
print("# Show config summary")
|
||||
print("python3 config_snapshot_downloader.py --config snapshot_config.json --show-config")
|
||||
print(
|
||||
"python3 config_snapshot_downloader.py --config snapshot_config.json --show-config"
|
||||
)
|
||||
|
||||
print("\n📄 Features:")
|
||||
print("• Downloads all snapshots with pagination support")
|
||||
|
||||
Reference in New Issue
Block a user