repo restructure

scripts/crontab

@@ -0,0 +1,9 @@
+# ParentZone Downloaders Cron Schedule
+# Run both downloaders daily at 2:00 AM
+0 2 * * * /app/scripts/scheduler.sh >> /var/log/cron.log 2>&1
+
+# Keep cron log file from growing too large (weekly cleanup)
+0 3 * * 0 find /var/log -name "cron.log" -size +100M -exec truncate -s 50M {} \; 2>/dev/null || true
+
+# Cleanup old snapshot files (keep last 90 days)
+30 3 * * 0 find /app/data/snapshots -name "*.html" -mtime +90 -delete 2>/dev/null || true

scripts/scheduler.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# ParentZone Downloaders Daily Scheduler
+# This script runs both the config downloader and snapshot downloader
+
+LOG_DIR="/app/data/logs"
+LOG_FILE="$LOG_DIR/scheduler_$(date +%Y%m%d).log"
+SNAPSHOT_CONFIG_FILE="/app/config/snapshot_config.json"
+ASSET_CONFIG_FILE="/app/config/parentzone_config.json"
+
+# Create log directory if it doesn't exist
+mkdir -p "$LOG_DIR"
+
+# Function to log messages with timestamp
+log_message() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
+}
+
+# Function to run a command and log its output
+run_with_logging() {
+    local command="$1"
+    local description="$2"
+
+    log_message "Starting: $description"
+    log_message "Command: $command"
+
+    # Run the command and capture both stdout and stderr
+    if eval "$command" >> "$LOG_FILE" 2>&1; then
+        log_message "SUCCESS: $description completed successfully"
+        return 0
+    else
+        log_message "ERROR: $description failed with exit code $?"
+        return 1
+    fi
+}
+
+# Main execution
+log_message "=== ParentZone Downloaders Daily Run Started ==="
+
+# Check if config files exist
+if [ ! -f "$SNAPSHOT_CONFIG_FILE" ]; then
+    log_message "ERROR: Snapshot configuration file $SNAPSHOT_CONFIG_FILE not found"
+    exit 1
+fi
+
+if [ ! -f "$ASSET_CONFIG_FILE" ]; then
+    log_message "WARNING: Asset configuration file $ASSET_CONFIG_FILE not found, skipping asset downloader"
+    SKIP_ASSET_DOWNLOADER=true
+else
+    SKIP_ASSET_DOWNLOADER=false
+fi
+
+cd /app
+
+# Run config-based asset downloader
+if [ "$SKIP_ASSET_DOWNLOADER" = false ]; then
+    run_with_logging "python3 src/config_downloader.py --config $ASSET_CONFIG_FILE" "Config Asset Downloader"
+    asset_result=$?
+else
+    log_message "SKIPPED: Config Asset Downloader (configuration file not found)"
+    asset_result=0
+fi
+
+# Run config-based snapshot downloader
+run_with_logging "python3 src/config_snapshot_downloader.py --config $SNAPSHOT_CONFIG_FILE" "Config Snapshot Downloader"
+config_result=$?
+
+# Run regular snapshot downloader with environment variables
+if [ -n "$API_KEY" ]; then
+    run_with_logging "python3 src/snapshot_downloader.py --api-key $API_KEY --output-dir data/snapshots" "Snapshot Downloader (API Key)"
+    snapshot_result=$?
+elif [ -n "$EMAIL" ] && [ -n "$PASSWORD" ]; then
+    run_with_logging "python3 src/snapshot_downloader.py --email $EMAIL --password $PASSWORD --output-dir data/snapshots" "Snapshot Downloader (Email/Password)"
+    snapshot_result=$?
+else
+    log_message "WARNING: No authentication method provided via environment variables, skipping direct snapshot downloader"
+    snapshot_result=0
+fi
+
+# Summary
+log_message "=== Daily Run Summary ==="
+if [ "$SKIP_ASSET_DOWNLOADER" = false ]; then
+    if [ $asset_result -eq 0 ]; then
+        log_message "✓ Config Asset Downloader: SUCCESS"
+    else
+        log_message "✗ Config Asset Downloader: FAILED"
+    fi
+else
+    log_message "- Config Asset Downloader: SKIPPED"
+fi
+
+if [ $config_result -eq 0 ]; then
+    log_message "✓ Config Snapshot Downloader: SUCCESS"
+else
+    log_message "✗ Config Snapshot Downloader: FAILED"
+fi
+
+if [ $snapshot_result -eq 0 ]; then
+    log_message "✓ Snapshot Downloader: SUCCESS"
+else
+    log_message "✗ Snapshot Downloader: FAILED"
+fi
+
+# Cleanup old log files (keep only last 30 days)
+find "$LOG_DIR" -name "scheduler_*.log" -mtime +30 -delete 2>/dev/null || true
+
+log_message "=== ParentZone Downloaders Daily Run Completed ==="
+
+# Exit with error if any downloader failed
+if [ $asset_result -ne 0 ] || [ $config_result -ne 0 ] || [ $snapshot_result -ne 0 ]; then
+    exit 1
+fi
+
+exit 0

scripts/startup.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# ParentZone Downloader Startup Script
+# This script starts both cron and the web server
+
+set -e
+
+LOG_DIR="/app/data/logs"
+SNAPSHOTS_DIR="/app/data/snapshots"
+
+# Create directories if they don't exist
+mkdir -p "$LOG_DIR"
+mkdir -p "$SNAPSHOTS_DIR"
+
+# Function to log messages with timestamp
+log_message() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - STARTUP - $1" | tee -a "$LOG_DIR/startup.log"
+}
+
+log_message "=== ParentZone Downloader Starting ==="
+
+# Start cron daemon
+log_message "Starting cron daemon..."
+cron
+
+# Create log file for cron output
+touch /var/log/cron.log
+
+log_message "Cron daemon started"
+
+# Start web server in the background
+log_message "Starting web server on port 8080..."
+python3 src/webserver.py --host 0.0.0.0 --port 8080 --snapshots-dir "$SNAPSHOTS_DIR" &
+WEB_SERVER_PID=$!
+
+log_message "Web server started with PID: $WEB_SERVER_PID"
+log_message "Web interface available at http://localhost:8080"
+
+# Function to handle shutdown
+shutdown() {
+    log_message "=== Shutdown Signal Received ==="
+
+    if [ ! -z "$WEB_SERVER_PID" ]; then
+        log_message "Stopping web server (PID: $WEB_SERVER_PID)..."
+        kill "$WEB_SERVER_PID" 2>/dev/null || true
+        wait "$WEB_SERVER_PID" 2>/dev/null || true
+        log_message "Web server stopped"
+    fi
+
+    log_message "Stopping cron daemon..."
+    pkill cron 2>/dev/null || true
+
+    log_message "=== ParentZone Downloader Shutdown Complete ==="
+    exit 0
+}
+
+# Set up signal handlers
+trap shutdown SIGTERM SIGINT
+
+log_message "=== ParentZone Downloader Started Successfully ==="
+log_message "Services running:"
+log_message "  - Cron daemon (scheduled downloads)"
+log_message "  - Web server at http://0.0.0.0:8080"
+log_message "  - Log files in: $LOG_DIR"
+log_message "  - Snapshots in: $SNAPSHOTS_DIR"
+
+# Keep the container running and follow cron logs
+tail -f /var/log/cron.log &
+TAIL_PID=$!
+
+# Wait for any process to exit
+wait $WEB_SERVER_PID $TAIL_PID