larger cache span

Dockers/puppeteer-api/app/database.py

@@ -91,6 +91,8 @@ def get_cached_data(url, route):
     conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
     cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)  # Convert hours to seconds
+    if route == 'pagination':
+        cache_expiry = cache_expiry * 31
     cursor.execute(
         "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
         (url, route, cache_expiry)
@@ -123,22 +125,31 @@ def save_to_cache(url, route, data):
 # Function to clean up old cache entries
 def cleanup_old_cache_entries():
     try:
-        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)")
+        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
 
         # Calculate the timestamp for entries older than the expiry time
         expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
+        pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
 
-        # Get count of entries to be deleted
-        cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,))
-        count = cursor.fetchone()[0]
+        # Get count of entries to be deleted (non-pagination)
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+        count_non_pagination = cursor.fetchone()[0]
+
+        # Get count of pagination entries to be deleted
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
+        count_pagination = cursor.fetchone()[0]
+
+        # Delete old non-pagination entries
+        cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+
+        # Delete old pagination entries
+        cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
 
-        # Delete old entries
-        cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,))
         conn.commit()
         conn.close()
 
-        print(f"Cache cleanup completed: {count} entries removed")
+        print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
     except Exception as e:
         print(f"Error during cache cleanup: {e}")

Dockers/puppeteer-api/main.py

@@ -149,23 +149,32 @@ def save_to_cache(url, route, data):
 # Function to clean up old cache entries
 def cleanup_old_cache_entries():
     try:
-        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)")
+        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
 
         # Calculate the timestamp for entries older than the expiry time
         expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
+        pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
 
-        # Get count of entries to be deleted
-        cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,))
-        count = cursor.fetchone()[0]
+        # Get count of entries to be deleted (non-pagination)
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+        count_non_pagination = cursor.fetchone()[0]
+
+        # Get count of pagination entries to be deleted
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
+        count_pagination = cursor.fetchone()[0]
+
+        # Delete old non-pagination entries
+        cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+
+        # Delete old pagination entries
+        cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
 
-        # Delete old entries
-        cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,))
         conn.commit()
         conn.close()
 
-        print(f"Cache cleanup completed: {count} entries removed")
+        print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
     except Exception as e:
         print(f"Error during cache cleanup: {e}")