first version of gmail invoice extractor

Dockers/gmail-invoice-extractor/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+invoices/
+token.json
+credentials.json
+invoice_extractor.log

Dockers/gmail-invoice-extractor/Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.12-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libffi-dev \
+    libxml2 \
+    libxslt1-dev \
+    libjpeg-dev \
+    zlib1g-dev \
+    libpoppler-cpp-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set workdir
+WORKDIR /app
+
+# Copy requirements first (better caching)
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy source code
+COPY . .
+
+# Make invoices folder inside container
+RUN mkdir -p /app/invoices
+
+# Run script
+CMD ["python", "main.py"]

Dockers/gmail-invoice-extractor/README.md

@@ -0,0 +1,76 @@
+# Gmail Invoice Extractor
+
+This application automatically scans Gmail for emails containing PDF attachments, checks if they contain your company's VAT number, and saves them to a designated folder.
+
+## Features
+
+- 🔍 Scans Gmail for emails with PDF attachments
+- 📄 Extracts text from PDF files using OCR
+- 💾 Automatically saves valid invoices to a local folder
+- 📝 Comprehensive logging
+- 🐳 Docker support
+
+## Setup
+
+### 1. Gmail API Setup
+
+1. Go to the [Google Cloud Console](https://console.cloud.google.com/)
+2. Create a new project or select an existing one
+3. Enable the Gmail API
+4. Create credentials (OAuth 2.0 Client ID)
+5. Download the credentials file and save it as `credentials.json` in this directory
+
+### 2. Configuration
+
+1. Update the VAT number patterns in `config.yaml` to match your company's VAT numbers
+2. Modify the Gmail search query if needed (default: emails with PDF attachments)
+
+### 3. Running the Application
+
+#### Option A: Direct Python execution
+
+```bash
+pip install -r requirements.txt
+python main.py
+```
+
+#### Option B: Using Docker
+
+```bash
+docker build -t gmail-invoice-extractor .
+docker run -v $(pwd)/invoices:/app/invoices -v $(pwd)/credentials.json:/app/credentials.json gmail-invoice-extractor
+```
+
+## First Run
+
+On the first run, the application will:
+
+1. Open a browser window for Gmail authentication
+2. Ask you to authorize the application
+3. Save the authentication token for future runs
+
+## Configuration
+
+Edit `config.yaml` to customize:
+
+- VAT number patterns for your company
+- Gmail search query
+- Maximum messages to process
+- Save folder location
+- Logging level
+
+## Logs
+
+The application creates detailed logs in `invoice_extractor.log` and displays them in the console.
+
+## Security Notes
+
+- Keep your `credentials.json` and `token.json` files secure
+- Never commit these files to version control
+- The application only requests read-only access to Gmail
+
+## Troubleshooting
+
+- **Authentication issues**: Delete `token.json` and run again to re-authenticate
+- **No PDFs found**: Check your Gmail search query in `config.yaml`
+- **VAT numbers not detected**: Verify your VAT number patterns in `config.yaml`

Dockers/gmail-invoice-extractor/config.py

@@ -0,0 +1,39 @@
+import yaml
+import os
+from typing import List
+
+class Config:
+    def __init__(self, config_file: str = "config.yaml"):
+        self.config_file = config_file
+        self.load_config()
+
+    def load_config(self):
+        """Load configuration from YAML file."""
+        if os.path.exists(self.config_file):
+            with open(self.config_file, 'r') as f:
+                config = yaml.safe_load(f)
+        else:
+            # Default configuration
+            config = {
+                'vat_patterns': [
+                    'BE0?\\d{9}',
+                    'NL\\d{9}B\\d{2}',
+                    'DE\\d{9}',
+                    'FR[A-Z0-9]{2}\\d{9}',
+                    'GB\\d{9,12}'
+                ],
+                'gmail_query': 'has:attachment filename:pdf',
+                'max_messages': 100,
+                'save_folder': 'invoices',
+                'log_level': 'INFO'
+            }
+
+        self.vat_patterns = config.get('vat_patterns', [])
+        self.gmail_query = config.get('gmail_query', 'has:attachment filename:pdf')
+        self.max_messages = config.get('max_messages', 100)
+        self.save_folder = config.get('save_folder', 'invoices')
+        self.log_level = config.get('log_level', 'INFO')
+
+    def get_vat_patterns(self) -> List[str]:
+        """Get VAT number patterns as compiled regex patterns."""
+        return self.vat_patterns

Dockers/gmail-invoice-extractor/config.yaml

@@ -0,0 +1,23 @@
+# Gmail Invoice Extractor Configuration
+
+# VAT number patterns for your company
+# Add your actual VAT number patterns here
+vat_patterns:
+  - "BE0?\\d{9}" # Belgium
+  - "NL\\d{9}B\\d{2}" # Netherlands
+  - "DE\\d{9}" # Germany
+  - "FR[A-Z0-9]{2}\\d{9}" # France
+  - "GB\\d{9,12}" # UK
+  # Add more patterns as needed
+
+# Gmail search query (modify as needed)
+gmail_query: "has:attachment filename:pdf"
+
+# Maximum number of messages to process per run
+max_messages: 100
+
+# Folder to save invoices
+save_folder: "invoices"
+
+# Logging level (DEBUG, INFO, WARNING, ERROR)
+log_level: "INFO"

Dockers/gmail-invoice-extractor/main.py

@@ -0,0 +1,258 @@
+import os
+import re
+import base64
+import pdfplumber
+import logging
+from datetime import datetime, timedelta
+from typing import List, Tuple, Optional
+
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from google.auth.transport.requests import Request
+
+from config import Config
+
+# Load configuration
+config = Config()
+
+# Configure logging
+logging.basicConfig(
+    level=getattr(logging, config.log_level),
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('invoice_extractor.log'),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Gmail API scope for read-only with attachments
+SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']
+
+# Folder to save invoices
+SAVE_FOLDER = config.save_folder
+os.makedirs(SAVE_FOLDER, exist_ok=True)
+
+
+def get_last_month_date_range():
+    """Get the date range for the last month in Gmail query format."""
+    today = datetime.now()
+
+    # Calculate first day of current month
+    first_day_current = today.replace(day=1)
+
+    # Calculate first day of last month
+    if first_day_current.month == 1:
+        first_day_last_month = first_day_current.replace(year=first_day_current.year - 1, month=12)
+    else:
+        first_day_last_month = first_day_current.replace(month=first_day_current.month - 1)
+
+    # Calculate last day of last month
+    last_day_last_month = first_day_current - timedelta(days=1)
+
+    # Format dates for Gmail query (YYYY/MM/DD format)
+    start_date = first_day_last_month.strftime('%Y/%m/%d')
+    end_date = last_day_last_month.strftime('%Y/%m/%d')
+
+    logger.info(f"Scanning emails from {start_date} to {end_date}")
+
+    return f"after:{start_date} before:{end_date}"
+
+
+def gmail_authenticate():
+    """Authenticate with Gmail API using OAuth2."""
+    creds = None
+    if os.path.exists("token.json"):
+        creds = Credentials.from_authorized_user_file("token.json", SCOPES)
+
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            try:
+                creds.refresh(Request())
+                logger.info("Credentials refreshed successfully")
+            except Exception as e:
+                logger.error(f"Failed to refresh credentials: {e}")
+                creds = None
+
+        if not creds:
+            try:
+                flow = InstalledAppFlow.from_client_secrets_file("credentials.json", SCOPES)
+                creds = flow.run_local_server(port=0)
+                logger.info("New credentials obtained")
+            except Exception as e:
+                logger.error(f"Failed to obtain credentials: {e}")
+                raise
+
+        with open("token.json", "w") as token:
+            token.write(creds.to_json())
+        logger.info("Credentials saved to token.json")
+
+    return build("gmail", "v1", credentials=creds)
+
+
+def search_messages(service, base_query: str, max_results: int = None) -> List[dict]:
+    """Search for messages matching the query with date filtering."""
+    if max_results is None:
+        max_results = config.max_messages
+
+    # Add date filter for last month
+    date_filter = get_last_month_date_range()
+    full_query = f"{base_query} {date_filter}"
+
+    try:
+        results = service.users().messages().list(
+            userId="me",
+            q=full_query,
+            maxResults=max_results
+        ).execute()
+        messages = results.get("messages", [])
+        logger.info(f"Found {len(messages)} messages matching query: {full_query}")
+        return messages
+    except Exception as e:
+        logger.error(f"Failed to search messages: {e}")
+        return []
+
+
+def get_attachments(service, msg_id: str) -> List[Tuple[str, bytes]]:
+    """Extract PDF attachments from a message."""
+    try:
+        message = service.users().messages().get(userId="me", id=msg_id).execute()
+        parts = message.get("payload", {}).get("parts", [])
+        attachments = []
+
+        for part in parts:
+            if part.get("filename") and part["filename"].lower().endswith(".pdf"):
+                att_id = part["body"].get("attachmentId")
+                if att_id:
+                    try:
+                        att = service.users().messages().attachments().get(
+                            userId="me", messageId=msg_id, id=att_id
+                        ).execute()
+                        data = base64.urlsafe_b64decode(att["data"])
+                        attachments.append((part["filename"], data))
+                        logger.debug(f"Extracted attachment: {part['filename']}")
+                    except Exception as e:
+                        logger.error(f"Failed to extract attachment {part['filename']}: {e}")
+
+        return attachments
+    except Exception as e:
+        logger.error(f"Failed to get attachments for message {msg_id}: {e}")
+        return []
+
+
+def extract_text_from_pdf(pdf_bytes: bytes) -> str:
+    """Extract text from PDF bytes."""
+    text = ""
+    temp_file = f"temp_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
+
+    try:
+        with open(temp_file, "wb") as f:
+            f.write(pdf_bytes)
+
+        with pdfplumber.open(temp_file) as pdf:
+            for page_num, page in enumerate(pdf.pages):
+                page_text = page.extract_text()
+                if page_text:
+                    text += page_text + "\n"
+                logger.debug(f"Extracted text from page {page_num + 1}")
+
+        return text
+    except Exception as e:
+        logger.error(f"Failed to extract text from PDF: {e}")
+        return ""
+    finally:
+        if os.path.exists(temp_file):
+            os.remove(temp_file)
+
+
+def contains_vat_number(text: str) -> bool:
+    """Check if text contains any of the configured VAT number patterns."""
+    if not text:
+        return False
+
+    text_upper = text.upper()
+    vat_patterns = config.get_vat_patterns()
+
+    for pattern in vat_patterns:
+        if re.search(pattern, text_upper):
+            logger.info(f"Found VAT number match with pattern: {pattern}")
+            return True
+
+    return False
+
+
+def save_invoice(filename: str, pdf_bytes: bytes) -> bool:
+    """Save invoice PDF to the designated folder."""
+    try:
+        # Create filename with timestamp to avoid conflicts
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        name, ext = os.path.splitext(filename)
+        safe_filename = f"{name}_{timestamp}{ext}"
+        save_path = os.path.join(SAVE_FOLDER, safe_filename)
+
+        with open(save_path, "wb") as f:
+            f.write(pdf_bytes)
+
+        logger.info(f"✅ Saved invoice: {safe_filename}")
+        return True
+    except Exception as e:
+        logger.error(f"Failed to save invoice {filename}: {e}")
+        return False
+
+
+def main():
+    """Main function to process Gmail messages and extract invoices."""
+    logger.info("Starting Gmail Invoice Extractor")
+    logger.info(f"Configuration: {config.gmail_query}, max_messages: {config.max_messages}")
+
+    try:
+        # Authenticate with Gmail
+        service = gmail_authenticate()
+
+        # Search for messages with PDF attachments from last month
+        messages = search_messages(service, config.gmail_query)
+
+        if not messages:
+            logger.info("No messages found with PDF attachments from last month")
+            return
+
+        processed_count = 0
+        saved_count = 0
+
+        for msg in messages:
+            try:
+                msg_id = msg["id"]
+                attachments = get_attachments(service, msg_id)
+
+                for filename, pdf_bytes in attachments:
+                    processed_count += 1
+                    logger.info(f"Processing attachment: {filename}")
+
+                    # Extract text from PDF
+                    text = extract_text_from_pdf(pdf_bytes)
+
+                    if not text:
+                        logger.warning(f"No text extracted from {filename}")
+                        continue
+
+                    # Check for VAT number
+                    if contains_vat_number(text):
+                        if save_invoice(filename, pdf_bytes):
+                            saved_count += 1
+                    else:
+                        logger.info(f"❌ Skipped (no VAT match): {filename}")
+
+            except Exception as e:
+                logger.error(f"Error processing message {msg.get('id', 'unknown')}: {e}")
+                continue
+
+        logger.info(f"Processing complete. Processed: {processed_count}, Saved: {saved_count}")
+
+    except Exception as e:
+        logger.error(f"Fatal error in main: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

Dockers/gmail-invoice-extractor/requirements.txt

@@ -0,0 +1,5 @@
+google-api-python-client
+google-auth-httplib2
+google-auth-oauthlib
+pdfplumber
+PyYAML