larger cache span

Dockers/puppeteer-api/main.py

@@ -149,23 +149,32 @@ def save_to_cache(url, route, data):
 # Function to clean up old cache entries
 def cleanup_old_cache_entries():
     try:
-        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)")
+        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
 
         # Calculate the timestamp for entries older than the expiry time
         expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
+        pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
 
-        # Get count of entries to be deleted
-        cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,))
-        count = cursor.fetchone()[0]
+        # Get count of entries to be deleted (non-pagination)
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+        count_non_pagination = cursor.fetchone()[0]
+
+        # Get count of pagination entries to be deleted
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
+        count_pagination = cursor.fetchone()[0]
+
+        # Delete old non-pagination entries
+        cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+
+        # Delete old pagination entries
+        cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
 
-        # Delete old entries
-        cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,))
         conn.commit()
         conn.close()
 
-        print(f"Cache cleanup completed: {count} entries removed")
+        print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
     except Exception as e:
         print(f"Error during cache cleanup: {e}")