367 lines
14 KiB
Python
367 lines
14 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
Test Asset Tracking Functionality
|
||
|
|
|
||
|
|
This script tests the asset tracking system to ensure new assets are detected
|
||
|
|
and only new/modified assets are downloaded.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
from pathlib import Path
|
||
|
|
from datetime import datetime
|
||
|
|
import os
|
||
|
|
|
||
|
|
# Add the current directory to the path so we can import modules
|
||
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
|
||
|
|
from asset_tracker import AssetTracker
|
||
|
|
from auth_manager import AuthManager
|
||
|
|
from image_downloader import ImageDownloader
|
||
|
|
|
||
|
|
|
||
|
|
class AssetTrackingTester:
|
||
|
|
"""Test class for asset tracking functionality."""
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
"""Initialize the tester."""
|
||
|
|
self.logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
# Mock API data for testing
|
||
|
|
self.mock_assets_v1 = [
|
||
|
|
{
|
||
|
|
"id": "asset_001",
|
||
|
|
"name": "family_photo_1.jpg",
|
||
|
|
"updated": "2024-01-01T10:00:00Z",
|
||
|
|
"size": 1024000,
|
||
|
|
"mimeType": "image/jpeg"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "asset_002",
|
||
|
|
"name": "birthday_party.jpg",
|
||
|
|
"updated": "2024-01-02T15:30:00Z",
|
||
|
|
"size": 2048000,
|
||
|
|
"mimeType": "image/jpeg"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "asset_003",
|
||
|
|
"name": "school_event.png",
|
||
|
|
"updated": "2024-01-03T09:15:00Z",
|
||
|
|
"size": 1536000,
|
||
|
|
"mimeType": "image/png"
|
||
|
|
}
|
||
|
|
]
|
||
|
|
|
||
|
|
self.mock_assets_v2 = [
|
||
|
|
# Existing asset - unchanged
|
||
|
|
{
|
||
|
|
"id": "asset_001",
|
||
|
|
"name": "family_photo_1.jpg",
|
||
|
|
"updated": "2024-01-01T10:00:00Z",
|
||
|
|
"size": 1024000,
|
||
|
|
"mimeType": "image/jpeg"
|
||
|
|
},
|
||
|
|
# Existing asset - modified
|
||
|
|
{
|
||
|
|
"id": "asset_002",
|
||
|
|
"name": "birthday_party.jpg",
|
||
|
|
"updated": "2024-01-05T16:45:00Z", # Updated timestamp
|
||
|
|
"size": 2100000, # Different size
|
||
|
|
"mimeType": "image/jpeg"
|
||
|
|
},
|
||
|
|
# Existing asset - unchanged
|
||
|
|
{
|
||
|
|
"id": "asset_003",
|
||
|
|
"name": "school_event.png",
|
||
|
|
"updated": "2024-01-03T09:15:00Z",
|
||
|
|
"size": 1536000,
|
||
|
|
"mimeType": "image/png"
|
||
|
|
},
|
||
|
|
# New asset
|
||
|
|
{
|
||
|
|
"id": "asset_004",
|
||
|
|
"name": "new_vacation_photo.jpg",
|
||
|
|
"updated": "2024-01-06T14:20:00Z",
|
||
|
|
"size": 3072000,
|
||
|
|
"mimeType": "image/jpeg"
|
||
|
|
}
|
||
|
|
]
|
||
|
|
|
||
|
|
def test_basic_tracking(self):
|
||
|
|
"""Test basic asset tracking functionality."""
|
||
|
|
print("=" * 60)
|
||
|
|
print("TEST 1: Basic Asset Tracking")
|
||
|
|
print("=" * 60)
|
||
|
|
|
||
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||
|
|
tracker = AssetTracker(storage_dir=temp_dir)
|
||
|
|
|
||
|
|
print(f"Testing with temporary directory: {temp_dir}")
|
||
|
|
|
||
|
|
# Test with empty tracker
|
||
|
|
print("\n1. Testing new asset detection (empty tracker)...")
|
||
|
|
new_assets = tracker.get_new_assets(self.mock_assets_v1)
|
||
|
|
print(f" Found {len(new_assets)} new assets (expected: 3)")
|
||
|
|
assert len(new_assets) == 3, f"Expected 3 new assets, got {len(new_assets)}"
|
||
|
|
print(" ✅ All assets correctly identified as new")
|
||
|
|
|
||
|
|
# Simulate downloading first batch
|
||
|
|
print("\n2. Simulating download of first batch...")
|
||
|
|
for asset in self.mock_assets_v1:
|
||
|
|
filename = asset['name']
|
||
|
|
filepath = Path(temp_dir) / filename
|
||
|
|
|
||
|
|
# Create dummy file
|
||
|
|
filepath.write_text(f"Mock content for {asset['id']}")
|
||
|
|
|
||
|
|
# Mark as downloaded
|
||
|
|
tracker.mark_asset_downloaded(asset, filepath, True)
|
||
|
|
print(f" Marked as downloaded: {filename}")
|
||
|
|
|
||
|
|
# Test tracker stats
|
||
|
|
stats = tracker.get_stats()
|
||
|
|
print(f"\n3. Tracker statistics after first batch:")
|
||
|
|
print(f" Total tracked assets: {stats['total_tracked_assets']}")
|
||
|
|
print(f" Successful downloads: {stats['successful_downloads']}")
|
||
|
|
print(f" Existing files: {stats['existing_files']}")
|
||
|
|
|
||
|
|
# Test with same assets (should find no new ones)
|
||
|
|
print("\n4. Testing with same assets (should find none)...")
|
||
|
|
new_assets = tracker.get_new_assets(self.mock_assets_v1)
|
||
|
|
print(f" Found {len(new_assets)} new assets (expected: 0)")
|
||
|
|
assert len(new_assets) == 0, f"Expected 0 new assets, got {len(new_assets)}"
|
||
|
|
print(" ✅ Correctly identified all assets as already downloaded")
|
||
|
|
|
||
|
|
print("\n✅ Basic tracking test passed!")
|
||
|
|
|
||
|
|
def test_modified_asset_detection(self):
|
||
|
|
"""Test detection of modified assets."""
|
||
|
|
print("\n" + "=" * 60)
|
||
|
|
print("TEST 2: Modified Asset Detection")
|
||
|
|
print("=" * 60)
|
||
|
|
|
||
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||
|
|
tracker = AssetTracker(storage_dir=temp_dir)
|
||
|
|
|
||
|
|
# Simulate first batch download
|
||
|
|
print("1. Simulating initial download...")
|
||
|
|
for asset in self.mock_assets_v1:
|
||
|
|
filename = asset['name']
|
||
|
|
filepath = Path(temp_dir) / filename
|
||
|
|
filepath.write_text(f"Mock content for {asset['id']}")
|
||
|
|
tracker.mark_asset_downloaded(asset, filepath, True)
|
||
|
|
|
||
|
|
print(f" Downloaded {len(self.mock_assets_v1)} assets")
|
||
|
|
|
||
|
|
# Test with modified assets
|
||
|
|
print("\n2. Testing with modified asset list...")
|
||
|
|
new_assets = tracker.get_new_assets(self.mock_assets_v2)
|
||
|
|
print(f" Found {len(new_assets)} new/modified assets")
|
||
|
|
|
||
|
|
# Should detect 1 modified + 1 new = 2 assets
|
||
|
|
expected = 2 # asset_002 (modified) + asset_004 (new)
|
||
|
|
assert len(new_assets) == expected, f"Expected {expected} assets, got {len(new_assets)}"
|
||
|
|
|
||
|
|
# Check which assets were detected
|
||
|
|
detected_ids = [asset['id'] for asset in new_assets]
|
||
|
|
print(f" Detected asset IDs: {detected_ids}")
|
||
|
|
|
||
|
|
assert 'asset_002' in detected_ids, "Modified asset_002 should be detected"
|
||
|
|
assert 'asset_004' in detected_ids, "New asset_004 should be detected"
|
||
|
|
assert 'asset_001' not in detected_ids, "Unchanged asset_001 should not be detected"
|
||
|
|
assert 'asset_003' not in detected_ids, "Unchanged asset_003 should not be detected"
|
||
|
|
|
||
|
|
print(" ✅ Correctly identified 1 modified + 1 new asset")
|
||
|
|
print("✅ Modified asset detection test passed!")
|
||
|
|
|
||
|
|
def test_cleanup_functionality(self):
|
||
|
|
"""Test cleanup of missing files."""
|
||
|
|
print("\n" + "=" * 60)
|
||
|
|
print("TEST 3: Cleanup Functionality")
|
||
|
|
print("=" * 60)
|
||
|
|
|
||
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||
|
|
tracker = AssetTracker(storage_dir=temp_dir)
|
||
|
|
|
||
|
|
# Create some files and track them
|
||
|
|
print("1. Creating and tracking assets...")
|
||
|
|
filepaths = []
|
||
|
|
for asset in self.mock_assets_v1:
|
||
|
|
filename = asset['name']
|
||
|
|
filepath = Path(temp_dir) / filename
|
||
|
|
filepath.write_text(f"Mock content for {asset['id']}")
|
||
|
|
tracker.mark_asset_downloaded(asset, filepath, True)
|
||
|
|
filepaths.append(filepath)
|
||
|
|
print(f" Created and tracked: {filename}")
|
||
|
|
|
||
|
|
# Remove one file manually
|
||
|
|
print("\n2. Removing one file manually...")
|
||
|
|
removed_file = filepaths[1]
|
||
|
|
removed_file.unlink()
|
||
|
|
print(f" Removed: {removed_file.name}")
|
||
|
|
|
||
|
|
# Check stats before cleanup
|
||
|
|
stats_before = tracker.get_stats()
|
||
|
|
print(f"\n3. Stats before cleanup:")
|
||
|
|
print(f" Total tracked: {stats_before['total_tracked_assets']}")
|
||
|
|
print(f" Existing files: {stats_before['existing_files']}")
|
||
|
|
print(f" Missing files: {stats_before['missing_files']}")
|
||
|
|
|
||
|
|
# Run cleanup
|
||
|
|
print("\n4. Running cleanup...")
|
||
|
|
tracker.cleanup_missing_files()
|
||
|
|
|
||
|
|
# Check stats after cleanup
|
||
|
|
stats_after = tracker.get_stats()
|
||
|
|
print(f"\n5. Stats after cleanup:")
|
||
|
|
print(f" Total tracked: {stats_after['total_tracked_assets']}")
|
||
|
|
print(f" Existing files: {stats_after['existing_files']}")
|
||
|
|
print(f" Missing files: {stats_after['missing_files']}")
|
||
|
|
|
||
|
|
# Verify cleanup worked
|
||
|
|
assert stats_after['missing_files'] == 0, "Should have no missing files after cleanup"
|
||
|
|
assert stats_after['total_tracked_assets'] == len(self.mock_assets_v1) - 1, "Should have one less tracked asset"
|
||
|
|
|
||
|
|
print(" ✅ Cleanup successfully removed missing file metadata")
|
||
|
|
print("✅ Cleanup functionality test passed!")
|
||
|
|
|
||
|
|
async def test_integration_with_downloader(self):
|
||
|
|
"""Test integration with ImageDownloader."""
|
||
|
|
print("\n" + "=" * 60)
|
||
|
|
print("TEST 4: Integration with ImageDownloader")
|
||
|
|
print("=" * 60)
|
||
|
|
|
||
|
|
# Note: This test requires actual API credentials to work fully
|
||
|
|
# For now, we'll test the initialization and basic functionality
|
||
|
|
|
||
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||
|
|
print(f"1. Testing ImageDownloader with asset tracking...")
|
||
|
|
|
||
|
|
try:
|
||
|
|
downloader = ImageDownloader(
|
||
|
|
api_url="https://api.parentzone.me",
|
||
|
|
list_endpoint="/v1/media/list",
|
||
|
|
download_endpoint="/v1/media",
|
||
|
|
output_dir=temp_dir,
|
||
|
|
track_assets=True
|
||
|
|
)
|
||
|
|
|
||
|
|
# Check if asset tracker was initialized
|
||
|
|
if downloader.asset_tracker:
|
||
|
|
print(" ✅ Asset tracker successfully initialized in downloader")
|
||
|
|
|
||
|
|
# Test tracker stats
|
||
|
|
stats = downloader.asset_tracker.get_stats()
|
||
|
|
print(f" Initial stats: {stats['total_tracked_assets']} tracked assets")
|
||
|
|
else:
|
||
|
|
print(" ❌ Asset tracker was not initialized")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f" Error during downloader initialization: {e}")
|
||
|
|
|
||
|
|
print("✅ Integration test completed!")
|
||
|
|
|
||
|
|
def run_all_tests(self):
|
||
|
|
"""Run all tests."""
|
||
|
|
print("🚀 Starting Asset Tracking Tests")
|
||
|
|
print("=" * 80)
|
||
|
|
|
||
|
|
try:
|
||
|
|
self.test_basic_tracking()
|
||
|
|
self.test_modified_asset_detection()
|
||
|
|
self.test_cleanup_functionality()
|
||
|
|
asyncio.run(self.test_integration_with_downloader())
|
||
|
|
|
||
|
|
print("\n" + "=" * 80)
|
||
|
|
print("🎉 ALL TESTS PASSED!")
|
||
|
|
print("=" * 80)
|
||
|
|
return True
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f"\n❌ TEST FAILED: {e}")
|
||
|
|
import traceback
|
||
|
|
traceback.print_exc()
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
async def test_with_real_api():
|
||
|
|
"""Test with real API (requires authentication)."""
|
||
|
|
print("\n" + "=" * 60)
|
||
|
|
print("REAL API TEST: Asset Tracking with ParentZone API")
|
||
|
|
print("=" * 60)
|
||
|
|
|
||
|
|
# Test credentials
|
||
|
|
email = "tudor.sitaru@gmail.com"
|
||
|
|
password = "mTVq8uNUvY7R39EPGVAm@"
|
||
|
|
|
||
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||
|
|
print(f"Using temporary directory: {temp_dir}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Create downloader with asset tracking
|
||
|
|
downloader = ImageDownloader(
|
||
|
|
api_url="https://api.parentzone.me",
|
||
|
|
list_endpoint="/v1/media/list",
|
||
|
|
download_endpoint="/v1/media",
|
||
|
|
output_dir=temp_dir,
|
||
|
|
email=email,
|
||
|
|
password=password,
|
||
|
|
track_assets=True,
|
||
|
|
max_concurrent=2 # Limit for testing
|
||
|
|
)
|
||
|
|
|
||
|
|
print("\n1. First run - downloading all assets...")
|
||
|
|
await downloader.download_all_assets()
|
||
|
|
|
||
|
|
if downloader.asset_tracker:
|
||
|
|
stats1 = downloader.asset_tracker.get_stats()
|
||
|
|
print(f"\nFirst run statistics:")
|
||
|
|
print(f" Downloaded assets: {stats1['successful_downloads']}")
|
||
|
|
print(f" Failed downloads: {stats1['failed_downloads']}")
|
||
|
|
print(f" Total size: {stats1['total_size_mb']} MB")
|
||
|
|
|
||
|
|
print("\n2. Second run - should find no new assets...")
|
||
|
|
downloader.stats = {'total': 0, 'successful': 0, 'failed': 0, 'skipped': 0}
|
||
|
|
await downloader.download_all_assets()
|
||
|
|
|
||
|
|
if downloader.asset_tracker:
|
||
|
|
stats2 = downloader.asset_tracker.get_stats()
|
||
|
|
print(f"\nSecond run statistics:")
|
||
|
|
print(f" New downloads: {downloader.stats['successful']}")
|
||
|
|
print(f" Skipped (unchanged): {len(stats2.get('total_tracked_assets', 0))}")
|
||
|
|
|
||
|
|
print("\n✅ Real API test completed!")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f"❌ Real API test failed: {e}")
|
||
|
|
import traceback
|
||
|
|
traceback.print_exc()
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
"""Main test function."""
|
||
|
|
# Setup logging
|
||
|
|
logging.basicConfig(
|
||
|
|
level=logging.INFO,
|
||
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||
|
|
)
|
||
|
|
|
||
|
|
tester = AssetTrackingTester()
|
||
|
|
|
||
|
|
# Run unit tests
|
||
|
|
success = tester.run_all_tests()
|
||
|
|
|
||
|
|
# Ask user if they want to run real API test
|
||
|
|
if success and len(sys.argv) > 1 and sys.argv[1] == '--real-api':
|
||
|
|
print("\n" + "🌐 Running real API test...")
|
||
|
|
asyncio.run(test_with_real_api())
|
||
|
|
|
||
|
|
return 0 if success else 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
exit(main())
|