diff --git a/Dockers/puppeteer-api/app/database.py b/Dockers/puppeteer-api/app/database.py index 6061d8f..68731c8 100644 --- a/Dockers/puppeteer-api/app/database.py +++ b/Dockers/puppeteer-api/app/database.py @@ -91,6 +91,8 @@ def get_cached_data(url, route): conn = sqlite3.connect(DB_PATH) cursor = conn.cursor() cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds + if route == 'pagination': + cache_expiry = cache_expiry * 31 cursor.execute( "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?", (url, route, cache_expiry) @@ -123,22 +125,31 @@ def save_to_cache(url, route, data): # Function to clean up old cache entries def cleanup_old_cache_entries(): try: - print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)") + print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)") conn = sqlite3.connect(DB_PATH) cursor = conn.cursor() # Calculate the timestamp for entries older than the expiry time expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) + pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60) - # Get count of entries to be deleted - cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,)) - count = cursor.fetchone()[0] + # Get count of entries to be deleted (non-pagination) + cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,)) + count_non_pagination = cursor.fetchone()[0] + + # Get count of pagination entries to be deleted + cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,)) + count_pagination = cursor.fetchone()[0] + + # Delete old non-pagination entries + cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,)) + + # Delete old pagination entries + cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,)) - # Delete old entries - cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,)) conn.commit() conn.close() - print(f"Cache cleanup completed: {count} entries removed") + print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed") except Exception as e: print(f"Error during cache cleanup: {e}") \ No newline at end of file diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index 4ffc6e3..49403a0 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -149,23 +149,32 @@ def save_to_cache(url, route, data): # Function to clean up old cache entries def cleanup_old_cache_entries(): try: - print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)") + print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)") conn = sqlite3.connect(DB_PATH) cursor = conn.cursor() # Calculate the timestamp for entries older than the expiry time expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) + pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60) - # Get count of entries to be deleted - cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,)) - count = cursor.fetchone()[0] + # Get count of entries to be deleted (non-pagination) + cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,)) + count_non_pagination = cursor.fetchone()[0] + + # Get count of pagination entries to be deleted + cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,)) + count_pagination = cursor.fetchone()[0] + + # Delete old non-pagination entries + cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,)) + + # Delete old pagination entries + cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,)) - # Delete old entries - cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,)) conn.commit() conn.close() - print(f"Cache cleanup completed: {count} entries removed") + print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed") except Exception as e: print(f"Error during cache cleanup: {e}")