Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

larger cache span
Build and Push Docker Images / build-and-push (push) Successful in 7m38s
Details

Browse Source
This commit is contained in:
Bram
2025-05-16 12:00:01 +02:00
parent e77dcbdf74
commit c8ec96d448
2 changed files with 34 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/database.py
+18 -7
View File
@@ -91,6 +91,8 @@ def get_cached_data(url, route):
conn = sqlite3.connect(DB_PATH) conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor() cursor = conn.cursor()
cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds
if route == 'pagination':
cache_expiry = cache_expiry * 31
cursor.execute( cursor.execute(
"SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?", "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
(url, route, cache_expiry) (url, route, cache_expiry)
@@ -123,22 +125,31 @@ def save_to_cache(url, route, data):
# Function to clean up old cache entries # Function to clean up old cache entries
def cleanup_old_cache_entries(): def cleanup_old_cache_entries():
try: try:
print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)") print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
conn = sqlite3.connect(DB_PATH) conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor() cursor = conn.cursor()
# Calculate the timestamp for entries older than the expiry time # Calculate the timestamp for entries older than the expiry time
expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
# Get count of entries to be deleted # Get count of entries to be deleted (non-pagination)
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,)) cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
count = cursor.fetchone()[0] count_non_pagination = cursor.fetchone()[0]
# Get count of pagination entries to be deleted
cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
count_pagination = cursor.fetchone()[0]
# Delete old non-pagination entries
cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
# Delete old pagination entries
cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
# Delete old entries
cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,))
conn.commit() conn.commit()
conn.close() conn.close()
print(f"Cache cleanup completed: {count} entries removed") print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
except Exception as e: except Exception as e:
print(f"Error during cache cleanup: {e}") print(f"Error during cache cleanup: {e}")
Dockers/puppeteer-api/main.py
+16 -7
View File
@@ -149,23 +149,32 @@ def save_to_cache(url, route, data):
# Function to clean up old cache entries # Function to clean up old cache entries
def cleanup_old_cache_entries(): def cleanup_old_cache_entries():
try: try:
print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)") print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
conn = sqlite3.connect(DB_PATH) conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor() cursor = conn.cursor()
# Calculate the timestamp for entries older than the expiry time # Calculate the timestamp for entries older than the expiry time
expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
# Get count of entries to be deleted # Get count of entries to be deleted (non-pagination)
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,)) cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
count = cursor.fetchone()[0] count_non_pagination = cursor.fetchone()[0]
# Get count of pagination entries to be deleted
cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
count_pagination = cursor.fetchone()[0]
# Delete old non-pagination entries
cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
# Delete old pagination entries
cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
# Delete old entries
cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,))
conn.commit() conn.commit()
conn.close() conn.close()
print(f"Cache cleanup completed: {count} entries removed") print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
except Exception as e: except Exception as e:
print(f"Error during cache cleanup: {e}") print(f"Error during cache cleanup: {e}")