firewall.ai:
Email as an AI-Native OS
A 4-stage deterministic data pipeline designed to extract structured intelligence (finance, logistics, career) from unstructured chronological noise. Built entirely on local, air-gapped LLMs.
Architecture Overview
01 // The Premise & Data Engineering
What if email was invented today, as an AI-native platform? Current email clients are just chronological feeds of unstructured text. Important signals (a direct message from a recruiter, a critical 2FA code, an upcoming AWS bill) are buried beneath promotional spam and system notifications.
To stress-test this premise, I engineered a highly sophisticated synthetic data generator. It simulates a busy professional's life (Uber receipts, AWS bills, Sequoia intro emails) to flood the database and prove the pipeline can handle the noise.
"""
firewall.ai - Master Demo Seed
"What if email was invented today, as an AI-native platform?"
Built for: Wealthsimple AI Builder Application
Author: Martin
-----------------------------------------------------------------------------
Generates 60 high-fidelity emails across 8 categories + 2 cinematic PDFs.
Every intelligence output is engineered to render beautifully in the Vault UI.
No lorem ipsum. No placeholder strings. Only signal.
-----------------------------------------------------------------------------
"""
import os
import psycopg2
import uuid
import json
from datetime import datetime, timedelta
from dotenv import load_dotenv
from fpdf import FPDF
load_dotenv()
# Database configuration loaded from environment variables
DB_HOST = os.getenv("DB_HOST", "localhost")
DB_PORT = os.getenv("DB_PORT", "5432")
DB_NAME = os.getenv("DB_NAME", "postgres")
DB_USER = os.getenv("DB_USER", "postgres")
DB_PASS = os.getenv("DB_PASS")
# -----------------------------------------------------------------------------
# PDF GENERATION - Two dense, realistic documents for OCR extraction demo
# -----------------------------------------------------------------------------
class ReceiptPDF(FPDF):
"""
Custom PDF generator class for creating highly structured, realistic physical receipts.
Inherits from FPDF to override header generation.
"""
def header(self):
"""
Generates the standard header for the Official Tax Invoice.
"""
self.set_font('Arial', 'B', 15)
self.cell(80)
self.cell(30, 10, 'OFFICIAL TAX INVOICE', 0, 0, 'C')
self.ln(20)
def generate_cinematic_pdfs():
"""
Generates two dense, realistic PDFs that will be OCR-extracted by extractor.py.
PDF 1: Williams-Sonoma high-end kitchen purchase receipt (rich line items).
PDF 2: RBC Direct Investing quarterly portfolio statement (fintech-relevant).
Outputs:
Saves files to the local 'saved_attachments' directory.
"""
os.makedirs("saved_attachments", exist_ok=True)
# -- PDF 1: WILLIAMS-SONOMA RECEIPT --------------------------------------
ws_path = "saved_attachments/demo_receipt_123.pdf"
pdf = ReceiptPDF()
pdf.add_page()
pdf.set_font("Courier", 'B', 13)
pdf.cell(200, 10, txt="WILLIAMS-SONOMA - YORKDALE SHOPPING CENTRE", ln=1, align='C')
pdf.set_font("Courier", size=9)
pdf.cell(200, 5, txt="3401 Dufferin St, Toronto, ON M6A 2T9 | Store #0441", ln=1, align='C')
pdf.cell(200, 5, txt="Tel: (416) 781-3322 | www.williams-sonoma.com", ln=1, align='C')
pdf.ln(4)
pdf.set_font("Courier", size=10)
pdf.cell(200, 8, txt="=" * 52, ln=1, align='C')
pdf.cell(200, 6, txt=f"Transaction #: WS-20260228-004419", ln=1)
pdf.cell(200, 6, txt=f"Date: February 28, 2026 | Time: 14:22:07", ln=1)
pdf.cell(200, 6, txt=f"Cashier: Emily R. | Terminal: POS-03", ln=1)
pdf.cell(200, 8, txt="=" * 52, ln=1, align='C')
pdf.ln(3)
items = [
("All-Clad D3 12\" Stainless Skillet", 1, 199.95),
("Staub Cast Iron Cocotte 5.5QT Navy", 1, 399.00),
("OXO Good Grips 3-Piece Mixing Bowl Set", 1, 59.99),
("Breville Barista Express Espresso", 1, 849.95),
("Global G-2 8\" Chef's Knife", 1, 189.00),
("Silpat Premium Non-Stick Baking Mat x2", 2, 44.99),
("Lodge Cast Iron Seasoning Spray", 1, 11.99),
("Microplane Premium Classic Zester", 1, 18.99),
]
total = 0
pdf.set_font("Courier", size=10)
for name, qty, price in items:
line_total = qty * price
pdf.cell(200, 7, txt=f"{qty}x {name[:34]:<34} ${line_total:>8.2f}", ln=1)
total += line_total
pdf.ln(3)
pdf.cell(200, 8, txt="-" * 52, ln=1)
pdf.cell(200, 7, txt=f"{'SUBTOTAL':<42} ${total:>8.2f}", ln=1)
pdf.cell(200, 7, txt=f"{'HST (13%)':<42} ${total * 0.13:>8.2f}", ln=1)
pdf.set_font("Courier", 'B', 12)
pdf.cell(200, 10, txt=f"{'TOTAL CAD':<42} ${total * 1.13:>8.2f}", ln=1)
pdf.set_font("Courier", size=9)
pdf.ln(5)
pdf.cell(200, 5, txt="Payment: VISA **** 4821 | Approved", ln=1)
pdf.cell(200, 5, txt="Rewards Points Earned: 1,773 pts | Balance: 24,610 pts", ln=1)
pdf.ln(5)
pdf.set_font("Courier", 'I', 8)
pdf.cell(200, 5, txt="Returns accepted within 30 days with original receipt.", ln=1, align='C')
pdf.cell(200, 5, txt="Thank you for shopping at Williams-Sonoma.", ln=1, align='C')
pdf.output(ws_path)
print(f" [OK] PDF 1 generated: {ws_path}")
# -- PDF 2: RBC DIRECT INVESTING QUARTERLY STATEMENT ---------------------
rbc_path = "saved_attachments/demo_flyer_123.pdf"
pdf2 = FPDF()
pdf2.add_page()
pdf2.set_font("Arial", 'B', 18)
pdf2.set_text_color(0, 60, 120)
pdf2.cell(200, 16, txt="RBC Direct Investing", ln=1, align='C')
pdf2.set_font("Arial", 'B', 13)
pdf2.set_text_color(50, 50, 50)
pdf2.cell(200, 8, txt="Q4 2025 Portfolio Statement | Jan 1 - Dec 31, 2025", ln=1, align='C')
pdf2.set_font("Arial", size=10)
pdf2.set_text_color(100, 100, 100)
pdf2.cell(200, 6, txt="Account Holder: Martin Leclair | TFSA Account #: RBC-TFSA-882741", ln=1, align='C')
pdf2.ln(8)
pdf2.set_text_color(0, 0, 0)
pdf2.set_font("Arial", 'B', 11)
pdf2.set_fill_color(230, 240, 255)
pdf2.cell(200, 8, txt=" HOLDINGS SUMMARY", ln=1, fill=True)
pdf2.ln(3)
holdings = [
("XEQT.TO", "iShares Core Equity ETF", 120, 38.42, 4610.40, "+18.2%"),
("VFV.TO", "Vanguard S&P 500 Index ETF", 55, 131.75, 7246.25, "+24.7%"),
("MSFT", "Microsoft Corporation", 12, 412.30, 4947.60, "+31.1%"),
("NVDA", "NVIDIA Corporation", 18, 138.90, 2500.20, "+167.4%"),
("ENB.TO", "Enbridge Inc.", 80, 56.10, 4488.00, "+9.3%"),
("BNS.TO", "Bank of Nova Scotia", 45, 72.80, 3276.00, "+5.8%"),
("CASH", "High-Interest Savings (RBC)", 1, 4200.00, 4200.00, "+3.1%"),
]
pdf2.set_font("Courier", 'B', 9)
pdf2.cell(200, 7, txt=f"{'SYMBOL':<10} {'NAME':<35} {'QTY':>5} {'PRICE':>9} {'VALUE':>10} {'YTD':>8}", ln=1)
pdf2.set_font("Courier", size=9)
pdf2.cell(200, 5, txt="-" * 82, ln=1)
port_total = 0
for sym, name, qty, price, value, ytd in holdings:
pdf2.cell(200, 7, txt=f"{sym:<10} {name:<35} {qty:>5} ${price:>8.2f} ${value:>9.2f} {ytd:>8}", ln=1)
port_total += value
pdf2.ln(3)
pdf2.cell(200, 5, txt="-" * 82, ln=1)
pdf2.set_font("Courier", 'B', 10)
pdf2.cell(200, 8, txt=f"{'TOTAL PORTFOLIO VALUE':<55} ${port_total:>9,.2f}", ln=1)
pdf2.set_font("Courier", size=9)
pdf2.ln(8)
pdf2.set_font("Arial", 'B', 11)
pdf2.set_fill_color(230, 240, 255)
pdf2.cell(200, 8, txt=" ACTIVITY THIS QUARTER", ln=1, fill=True)
pdf2.ln(3)
pdf2.set_font("Courier", size=9)
activity = [
("Nov 14", "BUY", "NVDA", "6 shares @ $121.40", "$728.40"),
("Nov 28", "DIVIDEND", "ENB.TO", "Quarterly $0.8875/sh", "$71.00"),
("Dec 02", "BUY", "XEQT.TO", "20 shares @ $35.90", "$718.00"),
("Dec 15", "DIVIDEND", "BNS.TO", "Quarterly $1.06/sh", "$47.70"),
("Dec 20", "SELL", "SHOP.TO", "10 shares @ $128.40", "$1,284.00"),
("Dec 31", "BUY", "VFV.TO", "5 shares @ $128.90", "$644.50"),
]
for date, action, sym, detail, amount in activity:
pdf2.cell(200, 7, txt=f"{date} {action:<10} {sym:<10} {detail:<30} {amount:>10}", ln=1)
pdf2.ln(6)
pdf2.set_font("Arial", 'I', 8)
pdf2.set_text_color(130, 130, 130)
pdf2.multi_cell(200, 5, txt="This statement is generated for informational purposes only. Past performance does not guarantee future results. RBC Direct Investing is a division of Royal Bank of Canada. Member of the Canadian Investor Protection Fund.")
pdf2.output(rbc_path)
print(f" [OK] PDF 2 generated: {rbc_path}")
# -----------------------------------------------------------------------------
# DATABASE HELPERS
# -----------------------------------------------------------------------------
def insert_email(cur, sender, subject, body, days_ago, tag, data=None, hours_ago=0):
"""
Inserts a generated email into the PostgreSQL database with an aggressive,
highly verbose print statement to simulate real-time terminal ingestion.
Args:
cur (psycopg2.cursor): The active database cursor.
sender (str): The email address of the sender.
subject (str): The subject line of the email.
body (str): The raw body content of the email.
days_ago (int): Number of days in the past to backdate the email.
tag (str): The initial librarian tag for categorization.
data (dict, optional): The pre-extracted structured payload. Defaults to None.
hours_ago (int, optional): Additional hour offset for fine-tuning time. Defaults to 0.
Returns:
int: The primary key ID of the inserted email record.
"""
date = datetime.now() - timedelta(days=days_ago, hours=hours_ago)
cur.execute("""
INSERT INTO emails (message_id, sender, subject, body_text, received_at, librarian_tag, extracted_data)
VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING id
""", (
str(uuid.uuid4()), sender, subject, body, date, tag,
json.dumps(data) if data else None
))
inserted_id = cur.fetchone()[0]
# π’ THE "MATRIX" VERBOSE INGESTION LOG
print("\n" + "β" * 70)
print(f"π‘ [INGEST STREAM] ALLOCATING MEMORY FOR ID: {inserted_id} | {date.strftime('%Y-%m-%d %H:%M:%S')}")
print(f"π€ SENDER : {sender}")
print(f"π SUBJECT : {subject}")
print(f"π CONTENTS:\n{body}")
if data:
print(f"π§ PRE-SEED PAYLOAD: {json.dumps(data)}")
print("β" * 70)
return inserted_id
# -----------------------------------------------------------------------------
# THE SEED - 60 emails of pure signal
# -----------------------------------------------------------------------------
def seed_swamp(cur):
"""
Executes the ingestion of 60 highly curated emails into the database.
Splits the data into 9 specific domains to properly stress-test the pipeline.
Args:
cur (psycopg2.cursor): The active database cursor.
"""
# ==========================================================================
# TIER 1: HUMAN DIRECT MESSAGES (8)
# The crown jewels - 1-to-1 human communication that deserves full attention
# ==========================================================================
insert_email(cur,
"priya.sharma@wealthsimple.com", "Re: Onboarding - Quick Question",
"Hey Martin, loved your demo yesterday. Quick one - does your pipeline handle attachment parsing natively or do you invoke a separate OCR step? Trying to understand the architecture before Thursday's sync.",
0, None, {"action_required": "Reply before Thursday", "contact": "Priya Sharma - Wealthsimple"})
insert_email(cur,
"alex.chen@sequoiacap.com", "Intro: AI Infrastructure Opportunity",
"Martin - I came across your work through the TechTO network. We're backing a series of AI-native SaaS plays in the productivity space and your background is directly relevant. Would love 20 minutes this week.",
1, None, {"action_required": "Schedule intro call", "opportunity": "Sequoia Capital AI Infra"})
insert_email(cur,
"dad@leclair.ca", "Cottage Weekend - March Long",
"Hey, your mom and I are heading up to the Muskoka place the weekend of the 14th. Any chance you're free? Stephanie is coming with the kids. Just let us know.",
2, None, {})
insert_email(cur,
"landlord@highparkproperties.ca", "Unit 4B - 2026 Lease Renewal",
"Martin, please find the 2026 lease agreement attached. Rent is increasing from $2,650 to $2,720/month as per the Ontario Rent Increase Guideline. Please sign and return by March 7th.",
3, None, {"action_required": "Sign and return lease by March 7", "upcoming_charge": 2720.00})
insert_email(cur,
"jordan.wu@company.com", "Production Deploy - Need Sign-off",
"Martin, the pipeline changes are staged and tested. I need your go-ahead to merge to main and trigger the deploy. Latency is down 40ms on P99. Happy to walk you through it.",
0, None, {"action_required": "Approve production deploy"}, hours_ago=2)
insert_email(cur,
"natalie.brooks@company.com", "Design Review: Vault UI Feedback",
"Hey - walked through the Vault with the design team this morning. The StructuredRenderer needs a pass. Raw JSON blobs are showing up in two of the intel panels. Flagging before the demo recording.",
0, None, {"action_required": "Fix StructuredRenderer JSON rendering"}, hours_ago=1)
insert_email(cur,
"recruiter@stripe.com", "Stripe - Staff Engineer, AI Platform",
"Hi Martin, I'm on Stripe's talent team. We're building out our AI platform org and your systems design background is exactly what we're looking for. The role is remote-first with a competitive equity package. Interested?",
1, None, {"opportunity": "Stripe Staff Engineer AI Platform"})
insert_email(cur,
"david.okafor@company.com", "Urgent: Client Demo Prep",
"Martin - Enterprise demo is tomorrow at 2pm EST. They specifically asked to see the AI triage logic in action with real examples. Can you make sure the seed data is clean and the pipeline has been run end-to-end?",
0, None, {"action_required": "Run full pipeline before 2pm tomorrow"}, hours_ago=3)
# ==========================================================================
# TIER 2: AUTH / SECURITY KEYS (6)
# 2FA codes, login alerts, magic links - highest friction, must surface instantly
# ==========================================================================
insert_email(cur,
"no-reply@accounts.google.com", "Security Alert: New Sign-in",
"Your Google account was accessed from a new device. Windows 11, Chrome, Toronto ON. If this was you, no action needed.",
0, None, {"urgent": True, "source": "Google"}, hours_ago=1)
insert_email(cur,
"security@github.com", "GitHub 2FA Code",
"Your GitHub authentication code is: 847-293. This code expires in 10 minutes. Do not share this code with anyone.",
0, None, {"urgent": True, "source": "GitHub"}, hours_ago=0)
insert_email(cur,
"noreply@okta.com", "Corporate Password Reset",
"A password reset was requested for your account at 09:14 AM EST. Click the secure link to set a new password. Link expires in 30 minutes.",
0, None, {"urgent": True, "source": "Okta"})
insert_email(cur,
"magic@slack.com", "Slack Login Link",
"Here's your login link for the company Slack workspace. This link is single-use and expires in 15 minutes. Do not forward.",
0, None, {"urgent": True, "source": "Slack"})
insert_email(cur,
"alerts@aws.amazon.com", "AWS Root Account Sign-In",
"We detected a root account sign-in to your AWS account (Account ID: 482-910-3847) from IP 142.150.22.81 at 11:02 PM EST. If this was not you, secure your account immediately.",
0, None, {"urgent": True, "source": "AWS"})
insert_email(cur,
"no-reply@wealthsimple.com", "Wealthsimple Login Verification",
"Use this code to complete your login: 612 448. Valid for 5 minutes.",
0, None, {"urgent": True, "source": "Wealthsimple"})
# ==========================================================================
# TIER 3: FINANCE (7)
# High-signal spending, bills, and anomalies - feeds the burn rate engine
# ==========================================================================
insert_email(cur,
"receipts@uber.com", "Your Thursday Evening Ride",
"Trip from King & Spadina to 32 Wellington St E. Duration: 14 min. Total: $22.40 (incl. tip).",
1, None, {"total": 22.40, "vendor": "Uber", "category": "Transit"})
insert_email(cur,
"no-reply@ttc.ca", "TTC Presto Auto-Load",
"Your Presto card (ending 8821) was auto-loaded with $50.00. New balance: $68.45.",
2, None, {"total": 50.00, "vendor": "Presto / TTC", "category": "Transit"})
insert_email(cur,
"billing@rogers.com", "Rogers - February Statement Ready",
"Your bill is ready. Amount due: $128.75 by March 15, 2026. Includes unlimited data plan + Rogers Ignite TV.",
0, None, {"upcoming_charge": 128.75, "vendor": "Rogers", "due_date": "March 15, 2026"})
insert_email(cur,
"billing@aws.amazon.com", "AWS Invoice - February 2026",
"Your estimated bill for February 2026 is $312.48 USD. Services: EC2 ($198.20), RDS ($74.10), S3 ($18.50), Data Transfer ($21.68).",
0, None, {"upcoming_charge": 312.48, "vendor": "AWS", "currency": "USD"})
insert_email(cur,
"statements@rbcroyalbank.com", "RBC Visa - Statement Available",
"Your February statement is ready. Balance: $4,218.90. Minimum payment: $42.00 due March 22. Notable charges: Williams-Sonoma $1,990.72, Swish Bloor $148.00, Indigo $82.40.",
0, None, {"upcoming_charge": 4218.90, "vendor": "RBC Visa", "due_date": "March 22, 2026"})
insert_email(cur,
"no-reply@wealthsimple.com", "TFSA Contribution Limit Updated",
"Good news - your 2026 TFSA contribution room has been updated to $7,000. Current TFSA balance: $31,268.45. You have $5,200 in remaining contribution room for 2026.",
1, None, {"total": 0, "vendor": "Wealthsimple TFSA", "note": "Contribution room: $5,200 remaining"})
insert_email(cur,
"alerts@questrade.com", "Margin Alert: SHOP Position",
"Your SHOP.TO position is approaching the maintenance margin threshold. Current equity: $2,140. Required margin: $1,900. Please review your portfolio.",
0, None, {"urgent": True, "upcoming_charge": 0, "vendor": "Questrade", "note": "Margin call risk on SHOP.TO"})
# ==========================================================================
# TIER 4: GROCERY RECEIPTS (5 - 2 are dense multi-item)
# Feeds the cross-reference engine: receipt history vs. active flyer sales
# ==========================================================================
loblaws_items = [
{"item_name": "Kirkland Whole Bean Coffee", "price": 24.99},
{"item_name": "Organic Chicken Breast 1.2kg", "price": 16.49},
{"item_name": "Avocados (5-pack)", "price": 6.99},
{"item_name": "Baby Spinach 142g", "price": 4.49},
{"item_name": "Eggs Free-Range 12pk", "price": 7.29},
{"item_name": "Kerrygold Butter Unsalted", "price": 8.99},
{"item_name": "Oat Milk (Oatly 1L x2)", "price": 9.98},
{"item_name": "Greek Yogurt (LibertΓ© 750g)", "price": 6.49},
{"item_name": "Ezekiel Sprouted Bread", "price": 6.99},
{"item_name": "Wild Atlantic Salmon 400g", "price": 19.99},
{"item_name": "Organic Basmati Rice 2kg", "price": 9.99},
{"item_name": "Olive Oil (Kirkland 2L)", "price": 22.49},
]
insert_email(cur,
"receipts@loblaws.ca", "Loblaws Receipt - Feb 22",
"12 items. Subtotal: $145.17. HST: $18.87. Total: $164.04. PC Optimum Points Earned: 1,640.",
14, None, {"items": loblaws_items, "vendor": "Loblaws", "total": 164.04})
metro_items = [
{"item_name": "AAA Striploin Steak 600g", "price": 28.99},
{"item_name": "Almond Milk (Silk 1.89L)", "price": 5.49},
{"item_name": "Blueberries (pint)", "price": 4.99},
{"item_name": "Asparagus Bunch", "price": 4.49},
{"item_name": "Aged Cheddar (Black Diamond)", "price": 9.49},
{"item_name": "San Marzano Crushed Tomatoes", "price": 3.99},
{"item_name": "Rigatoni Pasta (Garofalo)", "price": 3.49},
{"item_name": "Garlic (3-pack)", "price": 2.49},
{"item_name": "Lemons (4-pack)", "price": 3.49},
{"item_name": "Himalayan Pink Salt", "price": 4.99},
]
insert_email(cur,
"receipts@metro.ca", "Metro Receipt - Feb 28",
"10 items. Subtotal: $71.40. HST: $2.14. Total: $73.54.",
7, None, {"items": metro_items, "vendor": "Metro", "total": 73.54})
insert_email(cur,
"receipts@costco.ca", "Costco Receipt - Feb 15",
"Member #: 112-847-229. Total: $218.64. Notable: Kirkland Olive Oil, Protein Powder, Paper Towels 12pk, Chicken Thighs Bulk.",
15, None, {"items": [
{"item_name": "Kirkland Olive Oil 3L", "price": 29.99},
{"item_name": "Whey Protein (Kirkland 2.4kg)", "price": 59.99},
{"item_name": "Paper Towels 12pk", "price": 22.99},
{"item_name": "Chicken Thighs Bulk 3kg", "price": 24.99},
], "vendor": "Costco", "total": 218.64})
insert_email(cur,
"noreply@doordash.com", "DoorDash Order Delivered - Pai Northern Thai",
"Your order from Pai Northern Thai Kitchen has been delivered. Items: Massaman Curry, Roti, Thai Iced Tea. Subtotal: $41.00. Delivery + tip: $12.80. Total: $53.80.",
3, None, {"items": [
{"item_name": "Massaman Curry", "price": 22.00},
{"item_name": "Roti", "price": 7.00},
{"item_name": "Thai Iced Tea", "price": 6.50},
], "vendor": "DoorDash / Pai Thai", "total": 53.80})
insert_email(cur,
"receipts@wholefoods.com", "Whole Foods Market Receipt",
"Store: Whole Foods Yorkville. Items: Vital Farms Eggs, Organic Greens, Cashew Butter, Lion's Mane Supplement. Total: $94.30.",
5, None, {"items": [
{"item_name": "Vital Farms Pasture Eggs", "price": 9.99},
{"item_name": "Organic Power Greens 1lb", "price": 8.99},
{"item_name": "Artisana Cashew Butter", "price": 18.99},
{"item_name": "Host Defense Lion's Mane", "price": 49.99},
], "vendor": "Whole Foods", "total": 94.30})
# ==========================================================================
# TIER 5: FLYERS / PROMOTIONS (6)
# The right-join dataset - cross-referenced against receipt history for deals
# ==========================================================================
flyers = [
("flyers@loblaws.ca", "Loblaws - This Week's Deals",
"Kirkland Whole Bean Coffee $18.99, Organic Chicken Breast $11.99/kg, Avocados 5-pack $4.99, Oat Milk 2-pack $7.49",
[
{"item_name": "Kirkland Whole Bean Coffee", "sale_price": 18.99},
{"item_name": "Organic Chicken Breast 1.2kg", "sale_price": 12.99},
{"item_name": "Avocados (5-pack)", "sale_price": 4.99},
{"item_name": "Oat Milk (Oatly 1L x2)", "sale_price": 7.49},
]),
("flyers@metro.ca", "Metro Specials - Valid Mar 1-7",
"AAA Striploin $21.99/kg, Aged Cheddar $6.99, Blueberries 2-for-$7, Asparagus $2.99",
[
{"item_name": "AAA Striploin Steak 600g", "sale_price": 21.99},
{"item_name": "Aged Cheddar (Black Diamond)", "sale_price": 6.99},
{"item_name": "Blueberries (pint)", "sale_price": 3.50},
{"item_name": "Asparagus Bunch", "sale_price": 2.99},
]),
("flyers@costco.ca", "Costco - Member Savings Mar 2026",
"Kirkland Olive Oil $24.99, Protein Powder $49.99, Chicken Thighs Bulk $19.99, Paper Towels $18.99",
[
{"item_name": "Kirkland Olive Oil 3L", "sale_price": 24.99},
{"item_name": "Whey Protein (Kirkland 2.4kg)", "sale_price": 49.99},
{"item_name": "Chicken Thighs Bulk 3kg", "sale_price": 19.99},
{"item_name": "Paper Towels 12pk", "sale_price": 18.99},
]),
("flyers@wholefoods.com", "Whole Foods Prime Member Deals",
"Vital Farms Eggs $7.99, Organic Greens $5.99, Lion's Mane 20% off",
[
{"item_name": "Vital Farms Pasture Eggs", "sale_price": 7.99},
{"item_name": "Organic Power Greens 1lb", "sale_price": 5.99},
{"item_name": "Host Defense Lion's Mane", "sale_price": 39.99},
]),
("flyers@sobeys.com", "Sobeys - Weekend Flash Sale",
"Ezekiel Bread $4.99, Greek Yogurt $3.99, Wild Salmon $15.99/400g, Kerrygold Butter $6.49",
[
{"item_name": "Ezekiel Sprouted Bread", "sale_price": 4.99},
{"item_name": "Greek Yogurt (LibertΓ© 750g)", "sale_price": 3.99},
{"item_name": "Wild Atlantic Salmon 400g", "sale_price": 15.99},
{"item_name": "Kerrygold Butter Unsalted", "sale_price": 6.49},
]),
("flyers@nofrills.ca", "No Frills - Unbeatable Prices",
"Eggs 12pk $4.99, Basmati Rice 2kg $6.99, Baby Spinach $2.49, Olive Oil 1L $7.99",
[
{"item_name": "Eggs Free-Range 12pk", "sale_price": 4.99},
{"item_name": "Organic Basmati Rice 2kg", "sale_price": 6.99},
{"item_name": "Baby Spinach 142g", "sale_price": 2.49},
{"item_name": "Olive Oil (Kirkland 2L)", "sale_price": 14.99},
]),
]
for sender, subject, body, items in flyers:
insert_email(cur, sender, subject, body, 0, None, {"items": items})
# ==========================================================================
# TIER 6: LOGISTICS (6)
# Shipping, deliveries, and return windows - hard date math
# ==========================================================================
insert_email(cur,
"shipments@amazon.ca", "Out for Delivery: Keychron Q1 Pro Keyboard",
"Your Keychron Q1 Pro QMK Mechanical Keyboard is out for delivery today. Estimated arrival: 2-6pm. Amazon Logistics Driver: Raj.",
0, None, {"status": "Out for Delivery", "vendor": "Amazon", "item": "Keychron Q1 Pro Keyboard"})
insert_email(cur,
"tracking@dhl.com", "Shipment Held at Customs - Action Required",
"Your DHL shipment from Berlin (Waybill: 8291047743) is held at Canadian customs. An import duty of $48.50 CAD is owed before release. Pay at dhl.com/ca to avoid return.",
0, None, {"status": "Customs Hold", "vendor": "DHL", "fee": 48.50, "action_required": "Pay $48.50 customs duty"})
insert_email(cur,
"orders@bestbuy.ca", "Your LG UltraWide 34\" Has Been Delivered",
"Your LG 34WN80C-B UltraWide Monitor was delivered and signed for on Feb 1, 2026. Return window: 30 days from delivery. Return by: March 3, 2026.",
28, None, {"status": "Delivered", "vendor": "Best Buy", "item": "LG UltraWide 34\" Monitor", "return_policy_days": 30})
insert_email(cur,
"tracking@canadapost.ca", "Delivered: Package Left at Front Door",
"Your Canada Post package (Tracking: 7281 4847 2910 0291) was delivered at 1:42 PM and left at the front door.",
1, None, {"status": "Delivered", "vendor": "Canada Post"})
insert_email(cur,
"notify@ups.com", "UPS Delivery Delayed - Weather",
"Due to severe winter weather in the Toronto region, your shipment (1Z999AA10123456784) has been delayed by 1 business day. New estimated delivery: March 3, 2026.",
0, None, {"status": "Delayed", "vendor": "UPS", "item": "Pending package"})
insert_email(cur,
"orders@apple.com", "Your AirPods Pro 2 Are On Their Way",
"Apple Order #W123884729. AirPods Pro (2nd gen) with USB-C. Estimated delivery: March 2, 2026. Carrier: Purolator. Return window: 15 days.",
2, None, {"status": "In Transit", "vendor": "Apple", "item": "AirPods Pro 2 USB-C", "return_policy_days": 15})
# ==========================================================================
# TIER 7: CAREER (7)
# Job applications, interviews, recruiter outreach - action-driven signals
# ==========================================================================
insert_email(cur,
"no-reply@greenhouse.io", "Application Received - Wealthsimple AI Builder",
"Thank you for applying to the AI Builder role at Wealthsimple. Your application has been received and is under review. All candidates hear back within 48 hours of the March 2 close.",
1, None, {"network_update": "Applied - Wealthsimple AI Builder"})
insert_email(cur,
"scheduling@workday.com", "Interview Confirmed - March 4, 10:00 AM EST",
"Your pitch-style interview with the Wealthsimple executive panel is confirmed for Tuesday, March 4 at 10:00 AM EST. You will have 20 minutes to present followed by Q&A.",
0, None, {"action_required": "Prepare and rehearse demo presentation for March 4"})
insert_email(cur,
"opportunities@ycombinator.com", "YC W26 Batch - AI Infra Companies Hiring",
"8 YC W26 companies are actively hiring ML engineers and AI builders. Highlights: Cognition AI (Devin), Firecrawl (YC S24), Skyvern Automation. Roles open now.",
0, None, {"opportunity": "YC W26 AI Companies - 8 open roles"})
insert_email(cur,
"recruiting@openai.com", "OpenAI - Applied Research Engineer",
"Hi Martin, your profile came up in our systems. We're growing the Applied Research team in San Francisco. The role involves deploying frontier models at scale. TC: $380K-$520K USD. Interested?",
2, None, {"opportunity": "OpenAI Applied Research Engineer - $380-520K USD"})
insert_email(cur,
"connect@angellist.com", "5 New AI-Native Startup Roles Match Your Profile",
"New matches this week: Cohere (Toronto, Senior MLE), Writer.com (Staff Eng), Nous Research (Founding Eng), Harvey AI (AI Platform), Imbue (Research Eng).",
1, None, {"opportunity": "5 new AI startup roles - Cohere, Writer, Harvey, Imbue"})
insert_email(cur,
"events@techto.com", "TechTO March - You're on the Guest List",
"You're confirmed for TechTO's March 10 event at MaRS Discovery District. 400 attendees, 8 speaker sessions, networking from 6pm. Your QR ticket is attached.",
3, None, {"network_update": "TechTO March 10 - MaRS Discovery District"})
insert_email(cur,
"alerts@linkedin.com", "Marcus Liu Was Promoted to VP Engineering at Shopify",
"Your connection Marcus Liu has been promoted to VP of Engineering at Shopify. This might be a good time to reach out and congratulate him.",
0, None, {"network_update": "Marcus Liu promoted to VP Eng @ Shopify - reach out"})
# ==========================================================================
# TIER 8: NOTIFICATIONS (7)
# System pings, alerts, mentions - triaged by urgency
# ==========================================================================
insert_email(cur,
"notifications@github.com", "You Were Mentioned in PR #581 - firewall-ai/core",
"@martin-leclair - can you review the changes to intelligence_worker.py? I refactored the upsert logic and want your sign-off before we merge to main.",
0, None, {"urgent": True, "source": "GitHub", "task": "Review PR #581 - intelligence_worker.py refactor"})
insert_email(cur,
"alerts@datadoghq.com", "P99 Latency Spike - inference-api (Production)",
"[CRITICAL] P99 latency on inference-api has exceeded 2,400ms (threshold: 800ms) for the past 14 minutes. Triggered 3 retries. Investigate: https://app.datadoghq.com/apm/traces",
0, None, {"urgent": True, "source": "Datadog", "task": "Investigate P99 latency spike on inference-api"})
insert_email(cur,
"notify@jira.atlassian.com", "Jira: [FIRE-112] Assigned to You",
"[FIRE-112] - 'Pipeline fails silently when Ollama returns malformed JSON'. Priority: High. Sprint: March 1 Release. Assigned by: Jordan Wu.",
0, None, {"urgent": True, "source": "Jira", "task": "Fix silent pipeline failure on malformed Ollama JSON"})
insert_email(cur,
"notify@sentry.io", "New Issue: UnhandledPromiseRejection in /api/send",
"A new error has been detected in production: UnhandledPromiseRejection at POST /api/send. Occurred 7 times in the last hour. First seen: 2026-02-28 08:41 EST.",
0, None, {"urgent": True, "source": "Sentry", "task": "Fix UnhandledPromiseRejection in /api/send endpoint"})
insert_email(cur,
"notifications@slack.com", "Slack Digest - #ai-team",
"While you were away: 12 new messages in #ai-team. Jordan posted the revised system architecture diagram. Natalie shared the Figma design handoff for the Vault UI v2.",
0, None, {"urgent": False, "source": "Slack", "task": "Check #ai-team - architecture diagram + Vault UI v2 handoff"})
insert_email(cur,
"alerts@vercel.com", "Deployment Failed - firewall-ai (main)",
"Your Vercel deployment for firewall-ai failed on commit 3f8a22c. Build error: 'Module not found: react-pdf'. Check build logs at vercel.com/martin/firewall-ai.",
0, None, {"urgent": True, "source": "Vercel", "task": "Fix broken Vercel build - missing react-pdf module"})
insert_email(cur,
"updates@posthog.com", "Weekly Product Analytics Digest",
"This week: 142 active sessions (+28% WoW). Top feature: Vault Intelligence View (68% of sessions). Search used in 44% of sessions. Avg. session: 4m 12s.",
0, None, {"urgent": False, "source": "PostHog", "task": "Review session analytics - Vault is the top feature"})
# ==========================================================================
# TIER 9: NEWSLETTERS (7)
# Fintech-grade signal - frequency-counted and synthesized into headlines
# ==========================================================================
newsletters = [
("dan@tldrnewsletter.com", "TLDR - February 28, 2026",
"OpenAI drops GPT-5 with native tool use and 200K context. Stripe confirms Q2 2026 IPO at $65B valuation. Anthropic raises $3.5B Series E. Canada passes Bill C-27 AI regulation.",
["OpenAI GPT-5 launch", "Stripe IPO confirmed Q2 2026", "Anthropic $3.5B Series E", "Canada C-27 AI regulation"]),
("matt@bloomberg.com", "Bloomberg Money Stuff - Feb 28",
"Stripe's S-1 filing reveals $4.2B in net revenue for 2025, up 38% YoY. The IPO is expected to be the largest tech listing since Snowflake in 2020. Also: Citadel's new quant AI fund.",
["Stripe IPO S-1 filing - $4.2B revenue", "Largest tech IPO since Snowflake", "Citadel quant AI hedge fund"]),
("gergely@pragmaticengineer.com", "The Pragmatic Engineer - Feb 27",
"How Wealthsimple rebuilt its core trading infrastructure on Elixir. The shift from microservices to modular monoliths is accelerating. Cursor AI reaches 1M paying developers.",
["Wealthsimple trading infra rebuilt in Elixir", "Microservices to modular monolith trend", "Cursor AI 1M developers"]),
("crew@morningbrew.com", "Morning Brew - Feb 28",
"DeepSeek V3 benchmarks are forcing a rethink of AI infrastructure cost models. Canada's tech sector added 18,000 jobs in January. RBC launches AI-powered wealth advisor.",
["DeepSeek V3 cost model disruption", "Canada tech sector +18K jobs", "RBC AI wealth advisor launch"]),
("ben@benevansnewsletter.com", "Benedict Evans - The Next Step",
"AI agents are moving from demos to deployment. The key bottleneck isn't model capability - it's trust, auditability, and human-in-the-loop design. Who decides what the agent is allowed to do?",
["AI agents deployment bottleneck: trust and auditability", "Human-in-the-loop AI system design", "OpenAI GPT-5 launch"]),
("packy@notboring.co", "Not Boring - AI-Native Companies",
"The most interesting companies being built right now are not AI companies - they are legacy workflow companies being rebuilt from scratch with AI as the default. Stripe IPO momentum continues.",
["AI-native workflow rebuilds replacing legacy software", "Stripe IPO confirmed Q2 2026", "Cursor AI 1M developers"]),
("sarah@lennysnewsletter.com", "Lenny's Newsletter - Product + AI",
"How the best product teams are using AI to 10x their research velocity. The role of the PM is shifting: less Jira grooming, more system design. OpenAI GPT-5 changes the prototyping game.",
["AI research velocity - 10x product teams", "PM role shifting to system design", "OpenAI GPT-5 launch"]),
]
for sender, subject, body, topics in newsletters:
insert_email(cur, sender, subject, body, 0, None, {"topics": topics})
# ==========================================================================
# OCR TRIGGER ATTACHMENTS - Links emails to the generated PDFs
# These trigger extractor.py to OCR and extract structured line items
# ==========================================================================
ws_id = insert_email(cur,
"receipts@williams-sonoma.com", "Your Williams-Sonoma E-Receipt - Feb 28",
"Thank you for your purchase at Williams-Sonoma Yorkdale. Your e-receipt is attached. Total: $1,990.72 CAD. Returns accepted within 30 days.",
0, None, None)
cur.execute(
"INSERT INTO attachments (email_id, file_name, file_type, file_hash) VALUES (%s, %s, %s, %s)",
(ws_id, "WilliamsSonoma_Receipt_Feb28.pdf", "application/pdf", "demo_receipt_123")
)
rbc_id = insert_email(cur,
"statements@rbcdirectinvesting.com", "RBC Direct Investing - Q4 2025 Statement",
"Your Q4 2025 portfolio statement is ready. Total portfolio value: $31,268.45 CAD. Please review your holdings and year-end activity.",
1, None, None)
cur.execute(
"INSERT INTO attachments (email_id, file_name, file_type, file_hash) VALUES (%s, %s, %s, %s)",
(rbc_id, "RBC_DI_Q4_2025_Statement.pdf", "application/pdf", "demo_flyer_123")
)
print(f"\n [OK] 60 emails seeded across 9 categories.")
print(f" [OK] 2 PDF attachments linked for OCR extraction.")
# -----------------------------------------------------------------------------
# MAIN
# -----------------------------------------------------------------------------
def main():
"""
Main execution sequence for the demo seeding script.
Coordinates database connection, data wiping, and data generation.
"""
print("\n" + "=" * 70)
print(" firewall.ai - Epic Demo Seed")
print(" \"What if email was invented today, as an AI-native platform?\"")
print("=" * 70 + "\n")
print("[PDF] Generating cinematic PDFs...\n")
generate_cinematic_pdfs()
print("\n[SEED] Connecting to database and seeding swamp...\n")
try:
conn = psycopg2.connect(
host=DB_HOST, port=DB_PORT,
database=DB_NAME, user=DB_USER, password=DB_PASS
)
conn.autocommit = True
cur = conn.cursor()
print(" [WIPE] Wiping existing data...")
cur.execute("TRUNCATE TABLE emails CASCADE;")
seed_swamp(cur)
print("\n" + "=" * 70)
print(" [OK] Database seeded. firewall.ai is ready for its close-up.")
print(" >> Run the pipeline: python run_engine.py")
print("=" * 70 + "\n")
except Exception as e:
print(f"\n[ERR] Seed failed: {e}")
raise
finally:
try:
cur.close()
conn.close()
except:
pass
if __name__ == "__main__":
main()
02 // Stage 1 & 2: Triage & Vector Memory
The pipeline begins with a strict Primary Triage Engine running on a local Qwen3 model. It applies negative constraints to isolate 1-to-1 human communication and Auth Keys from general operational data.
"""
Primary Triage Engine (Stage 1 Routing)
This module performs the initial classification of incoming unstructured email streams.
It isolates high-signal, zero-friction communications (Direct Messages and
Authentication Events) from the general operational data stream (Vault items),
updating the PostgreSQL state for downstream consumption.
"""
import os
import json
import psycopg2
from dotenv import load_dotenv
import ollama
load_dotenv()
DB_HOST = os.getenv("DB_HOST", "localhost")
DB_NAME = os.getenv("DB_NAME", "postgres")
DB_USER = os.getenv("DB_USER", "postgres")
DB_PASS = os.getenv("DB_PASS")
SYSTEM_PROMPT = """
You are an intelligent email triage engine.
FIRST PRINCIPLE 1: THE DEFINITION OF 'HUMAN' (is_human)
- TRUE ONLY IF: Bespoke, 1-to-1 communication typed by a specific person (Coworker, Recruiter, Landlord, Family).
- NEGATIVE CONSTRAINTS (IS NOT HUMAN):
* If the sender is a brand or service (Uber, Starbucks, Apple, AWS, Google).
* If it is a Newsletter or Digest (TLDR, Money Stuff, Pragmatic Engineer, Morning Brew, Benedict Evans).
* If it is an automated notification (GitHub, Jira, Slack, LinkedIn).
* Even if it has a person's name in the 'From' field, if the content is a mass-distribution article, it is FALSE.
FIRST PRINCIPLE 2: THE DEFINITION OF A 'KEY' (is_key)
- TRUE ONLY IF: Security Alerts, 2FA, Password Resets, or Magic Login Links.
- NEGATIVE CONSTRAINTS (IS NOT A KEY):
* Receipts are NOT keys.
* Billing alerts are NOT keys.
* Shipping updates are NOT keys.
OUTPUT EXACTLY IN THIS JSON FORMAT:
{
"is_human": boolean,
"is_key": boolean,
"summary": "Summary"
}
"""
def sort_emails():
"""
Connects to the database to fetch unprocessed emails, routes them through
a local LLM for initial binary classification, and idempotently updates
their triage status in the database.
"""
print("[INIT] [STAGE 1] Initializing Primary Triage Engine (Direct/Auth Routing)...")
conn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = conn.cursor()
try:
cur.execute("SELECT id, sender, subject, body_text FROM emails WHERE is_processed = FALSE;")
emails = cur.fetchall()
for email_id, sender, subject, body in emails:
content = f"Sender: {sender}\nSubject: {subject}\nBody: {body[:4000]}"
try:
response = ollama.chat(model='qwen3:8b-q4_K_M', messages=[
{'role': 'system', 'content': SYSTEM_PROMPT},
{'role': 'user', 'content': content}
], format='json')
pred = json.loads(response['message']['content'])
is_human = pred.get('is_human', False)
is_key = pred.get('is_key', False)
cur.execute(
"UPDATE emails SET is_human=%s, is_key=%s, summary_text=%s, is_processed=TRUE WHERE id=%s",
(is_human, is_key, pred.get('summary', ''), email_id)
)
if is_key:
print(f" [AUTH] | {subject[:50]}")
elif is_human:
print(f" [DM] | {subject[:50]}")
else:
print(f" [VAULT] | {subject[:50]}")
except Exception:
pass
conn.commit()
finally:
cur.close()
conn.close()
if __name__ == "__main__":
sort_emails()
The remaining "noise" moves to Stage 2, where it is routed into 7 taxonomic Vaults (Finance, Logistics, Newsletters). Simultaneously, we generate dense vector representations using all-mpnet-base-v2 and push them to PostgreSQL (pgvector) to power the natural language search UI.
"""
Secondary Classification & Vector Memory Engine (Stage 2)
This module processes the operational data stream (noise) that bypassed
the Primary Triage Engine. It executes two critical pipeline steps:
1. Taxonomy Routing: Uses a local LLM to deterministically map unstructured
emails into one of 7 predefined intelligence categories (Vault tabs).
2. Semantic Embedding: Generates dense vector representations (all-mpnet-base-v2)
of the processed data and pushes them to PostgreSQL (pgvector) for retrieval.
"""
import os
import json
import psycopg2
from dotenv import load_dotenv
import ollama
from sentence_transformers import SentenceTransformer
load_dotenv()
DB_HOST = os.getenv("DB_HOST", "localhost")
DB_NAME = os.getenv("DB_NAME", "postgres")
DB_USER = os.getenv("DB_USER", "postgres")
DB_PASS = os.getenv("DB_PASS")
LIBRARIAN_PROMPT = """
You are a highly structured classification engine. Categorize the email into EXACTLY ONE of these tags:
- "LOGISTICS": Item shipments, tracking, and receipts for physical hardware with return policies (e.g., BestBuy, Amazon).
- "FINANCE": High-value anomalies, banking alerts, dividends, and cloud infrastructure bills (e.g., AWS billing).
- "NEWSLETTER": Tech/finance digests, Substacks, and news roundups. NEVER tag these as Finance.
- "RECEIPT": Consumable food, transit, or basic digital receipts (e.g., Uber, Starbucks, Loblaws, Costco).
- "CAREER": Application updates, interview scheduling, startup alerts.
- "PROMOTION": Retail sales, marketing flyers, weekly deals (e.g., Walmart Rollback, Loblaws Flyer).
- "NOTIFICATION": Ping alerts, Datadog, GitHub, Slack, Jira, LinkedIn network updates.
OUTPUT EXACTLY IN THIS JSON FORMAT AND NOTHING ELSE:
{ "category": "TAG" }
"""
def deploy_librarian():
"""Executes the secondary classification and semantic embedding pipeline."""
print("[INIT] [STAGE 2] Initializing Vault Classification Engine...")
model = SentenceTransformer('all-mpnet-base-v2')
conn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = conn.cursor()
try:
cur.execute("""
SELECT id, subject, body_text
FROM emails
WHERE is_processed = TRUE
AND is_human = FALSE
AND is_key = FALSE
AND librarian_tag IS NULL;
""")
emails_to_tag = cur.fetchall()
for email_id, subject, body in emails_to_tag:
try:
response = ollama.chat(model='qwen3:8b-q4_K_M', messages=[
{'role': 'system', 'content': LIBRARIAN_PROMPT},
{'role': 'user', 'content': f"Subject: {subject}\nBody: {body[:4000]}"}
], format='json')
tag = json.loads(response['message']['content']).get('category', 'NOTIFICATION')
valid_tags = ["RECEIPT", "CAREER", "FINANCE", "LOGISTICS", "NEWSLETTER", "PROMOTION", "NOTIFICATION"]
if tag not in valid_tags:
tag = "NOTIFICATION"
cur.execute("UPDATE emails SET librarian_tag = %s WHERE id = %s", (tag, email_id))
print(f" [TAG: {tag:<12}] | {subject[:45]}...")
except Exception:
cur.execute("UPDATE emails SET librarian_tag = 'NOTIFICATION' WHERE id = %s", (email_id,))
conn.commit()
cur.execute("""
SELECT e.id, e.subject, e.summary_text, coalesce(e.librarian_tag, 'SIGNAL')
FROM emails e
LEFT JOIN embeddings em ON e.id = em.email_id
WHERE em.id IS NULL
AND e.is_processed = TRUE;
""")
emails_to_embed = cur.fetchall()
if emails_to_embed:
print("\n[INFO] [STAGE 2.5] Generating Semantic Embeddings (all-mpnet-base-v2)...")
for email_id, subject, summary, tag in emails_to_embed:
text = f"Tag: {tag}\nSubject: {subject}\nSummary: {summary}"
vector = model.encode(text).tolist()
cur.execute(
"INSERT INTO embeddings (email_id, content_chunk, embedding) VALUES (%s, %s, %s)",
(email_id, text, vector)
)
print(f" [VECTOR] | ID: {email_id} -> Pushed to pgvector")
conn.commit()
finally:
cur.close()
conn.close()
if __name__ == "__main__":
deploy_librarian()
03 // Stage 3: Dual-Path Document Ingestion
Real intelligence lives in attachments. A standard API wrapper fails here. I engineered a Dual-Path Ingestion system: unstructured physical PDFs (like a Williams-Sonoma receipt) are parsed via pypdf, and the text layer is pushed through the LLM.
The critical step is forcing the LLM to normalize this raw text into the exact same strict JSON schema used by HTML email ingestion.
"""
Document OCR & Structure Extraction Engine
This module handles the "Dual-Path Ingestion" architecture. It queries the database
for unstructured physical documents (PDFs) attached to emails, performs local text
extraction, and leverages a local LLM to normalize the raw text into the exact same
strict JSON schema used by the HTML email ingestion path.
"""
import os
import json
import psycopg2
from dotenv import load_dotenv
import ollama
from pypdf import PdfReader
load_dotenv()
DB_HOST = os.getenv("DB_HOST", "localhost")
DB_PORT = os.getenv("DB_PORT", "5432")
DB_NAME = os.getenv("DB_NAME", "postgres")
DB_USER = os.getenv("DB_USER", "postgres")
DB_PASS = os.getenv("DB_PASS")
def extract_text_from_pdf(pdf_path):
"""Reads a physical PDF file from disk and extracts its raw text layer."""
try:
reader = PdfReader(pdf_path)
return "".join([page.extract_text() + "\n" for page in reader.pages])
except Exception as e:
print(f" [ERROR] [OCR] Failed to read PDF {pdf_path}: {e}")
return ""
EXTRACTION_PROMPT = """
You are a highly precise data extraction AI handling BOTH receipts and retail flyers.
Extract the vendor name, the grand total (if it's a receipt; use 0.0 if it's a flyer), and an itemized list of up to 5 main items from the text.
Return ONLY a valid JSON object in this exact format. Separate the item name from the price or sale price:
{
"vendor": "Store Name",
"total": 123.45,
"items": [
{"item_name": "Chicken Breast", "price": 12.99},
{"item_name": "Spinach", "price": 4.99}
]
}
"""
def run_extraction():
"""Coordinates the document extraction pipeline."""
print("[INIT] [STAGE 3] Initializing Document OCR & Structure Extraction Engine...")
conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = conn.cursor()
try:
cur.execute("""
SELECT e.id, a.file_hash, a.file_name
FROM emails e JOIN attachments a ON e.id = a.email_id
WHERE e.extracted_data IS NULL;
""")
receipts = cur.fetchall()
if not receipts:
return
save_dir = os.path.join(os.getcwd(), "saved_attachments")
for email_id, file_hash, file_name in receipts:
pdf_path = os.path.join(save_dir, f"{file_hash}.pdf")
if not os.path.exists(pdf_path):
print(f" [WARN] [OCR] Missing file on disk: {pdf_path}")
continue
raw_text = extract_text_from_pdf(pdf_path)
if not raw_text.strip():
print(f" [WARN] [OCR] No text layer found in {file_name}")
continue
try:
response = ollama.chat(model='qwen3:8b-q4_K_M', messages=[
{'role': 'system', 'content': EXTRACTION_PROMPT},
{'role': 'user', 'content': f"Receipt Text:\n{raw_text[:4000]}"}
], format='json')
extracted_json = json.loads(response['message']['content'])
vendor = extracted_json.get('vendor', 'Unknown')
total = extracted_json.get('total', 0.0)
cur.execute("UPDATE emails SET extracted_data = %s WHERE id = %s",
(json.dumps(extracted_json), email_id))
print(f" [OCR] | {vendor:<25} | ${total:.2f}")
except Exception:
pass
conn.commit()
finally:
cur.close()
conn.close()
if __name__ == "__main__":
run_extraction()
04 // Stage 4: Decoupling Math from LLMs
This is the "Chief of Staff" layer and the most important architectural decision. LLMs are terrible at math.
Instead of asking the LLM to summarize finances, the intelligence_worker.py uses pure deterministic Python to calculate bank balances, sum liabilities, and track return window dates. Probabilistic LLM logic is strictly reserved for generative tasksβlike identifying intersecting data between past grocery receipts and current flyers to generate gourmet meal plans.
"""
firewall.ai - Intelligence Worker (Modular Domain Engine)
This module acts as the "Chief of Staff" (Stage 3). It consumes the normalized,
categorized data from the Vault and performs domain-specific synthesis.
Crucially, it decouples deterministic logic (date math, sum calculations) from
probabilistic logic (LLM summarization, recipe generation) to ensure absolute
accuracy on financial and logistical operations.
Architecture:
run_domain(tag) -> Fetches emails for a specific tag, processes, and upserts.
run_all() -> Executes the full pipeline across all domains.
CLI: python intelligence_worker.py [DOMAIN_TAG]
"""
import os
import sys
import json
from datetime import datetime
from typing import Any, Dict, List, Tuple
from collections import Counter
import psycopg2
import ollama
from dotenv import load_dotenv
load_dotenv()
DB_HOST = os.getenv("DB_HOST", "localhost")
DB_PORT = os.getenv("DB_PORT", "5432")
DB_NAME = os.getenv("DB_NAME", "postgres")
DB_USER = os.getenv("DB_USER", "postgres")
DB_PASS = os.getenv("DB_PASS")
# =============================================================================
# SHARED HELPERS
# =============================================================================
def get_conn() -> psycopg2.extensions.connection:
"""Establishes and returns a connection to the PostgreSQL database."""
return psycopg2.connect(
host=DB_HOST, port=DB_PORT,
database=DB_NAME, user=DB_USER, password=DB_PASS
)
def extract_json(prompt: str, schema: str) -> Dict[str, Any]:
"""
Interfaces with the local Qwen3 model to enforce strict JSON synthesis.
Args:
prompt (str): The natural language instruction and context.
schema (str): The stringified JSON schema the LLM must adhere to.
Returns:
Dict[str, Any]: The parsed JSON object, or an empty dict on failure.
"""
try:
response = ollama.chat(
model="qwen3:8b-q4_K_M",
messages=[
{"role": "system", "content": f"You are a precise data synthesizer. Output ONLY valid JSON matching this schema: {schema}. No markdown, no explanation, no <think> tags."},
{"role": "user", "content": prompt}
],
format="json"
)
return json.loads(response["message"]["content"])
except Exception as e:
print(f" [WARN] LLM Synthesis failed: {e}")
return {}
def get_emails_by_tag(cur: psycopg2.extensions.cursor, tag: str) -> List[Tuple]:
"""Fetches all processed emails associated with a specific intelligence domain."""
cur.execute(
"SELECT id, sender, subject, received_at, extracted_data FROM emails WHERE librarian_tag = %s",
(tag,)
)
return cur.fetchall()
def fmt(val: Any) -> Any:
"""Formats Python dictionaries and lists into JSON strings for Postgres ingestion."""
return json.dumps(val) if isinstance(val, (dict, list)) else val
def upsert(cur: psycopg2.extensions.cursor, table: str, cols: List[str], vals: List[Any]) -> None:
"""
Performs an idempotent ON CONFLICT DO UPDATE operation to safely overwrite
intelligence records without creating duplicate rows for the current date.
"""
query = f"""
INSERT INTO {table} (report_date, {", ".join(cols)})
VALUES (CURRENT_DATE, {", ".join(["%s"] * len(cols))})
ON CONFLICT (report_date) DO UPDATE SET
{", ".join(f"{c} = EXCLUDED.{c}" for c in cols)};
"""
cur.execute(query, [fmt(v) for v in vals])
# =============================================================================
# DOMAIN: FINANCE
# =============================================================================
def process_finance(emails: List[Tuple]) -> Dict[str, Any]:
"""Isolates upcoming liabilities and bills from the finance data stream."""
print(" [FINANCE] Calculating deterministic liabilities...")
liabilities = []
for email_id, sender, subject, _, data in emails:
if not data:
continue
upcoming = data.get("upcoming_charge", 0.0) or 0.0
if upcoming > 0:
liabilities.append({
"vendor": data.get("vendor", sender),
"upcoming_charge": upcoming,
"due_date": data.get("due_date", ""),
"note": data.get("note", ""),
"source_id": email_id # PROVENANCE LINK
})
return {
"upcoming_liabilities": liabilities,
}
def run_finance(cur: psycopg2.extensions.cursor) -> None:
"""Executes the Finance domain pipeline."""
emails = get_emails_by_tag(cur, "FINANCE")
r = process_finance(emails)
upsert(cur, "intel_finance",
["upcoming_liabilities"],
[r["upcoming_liabilities"]])
print(f" β³ Liabilities: {len(r['upcoming_liabilities'])}")
# =============================================================================
# DOMAIN: GROCERIES (RECEIPT + PROMOTION)
# =============================================================================
def process_groceries(receipts: List[Tuple], flyers: List[Tuple]) -> Dict[str, Any]:
"""Cross-references historical purchase data against active promotional flyers."""
print(" [COMMERCE] Cross-referencing receipts vs. active deals...")
past_items: Dict[str, Dict] = {}
receipt_total = 0.0
for email_id, sender, subject, received_at, data in receipts:
if not data:
continue
receipt_total += data.get("total", 0.0) or 0.0
store_name = data.get("vendor", "") or sender.split("@")[1].split(".")[0].title()
receipt_date = received_at.strftime("%b %d") if received_at else ""
if "items" in data:
for item in data["items"]:
name = item.get("item_name", "").strip()
price = item.get("price", 0.0) or 0.0
if name and price > 0:
past_items[name] = {
"price": price,
"source_store": store_name,
"source_date": receipt_date,
"source_id": email_id # PROVENANCE LINK
}
re_up_map: Dict[str, Dict] = {}
alert_map: Dict[str, Dict] = {}
for email_id, sender, subject, received_at, data in flyers:
if not data or "items" not in data:
continue
deal_store = data.get("vendor", "") or sender.split("@")[1].split(".")[0].title()
deal_date = received_at.strftime("%b %d") if received_at else ""
for item in data["items"]:
name = item.get("item_name", "").strip()
sale_price = item.get("sale_price", 0.0) or 0.0
if not name or sale_price <= 0 or name not in past_items:
continue
history = past_items[name]
past_price = history["price"]
if name not in re_up_map or sale_price < re_up_map[name]["sale_price"]:
re_up_map[name] = {
"item": name,
"sale_price": round(sale_price, 2),
"deal_store": deal_store,
"deal_date": deal_date,
"deal_email_id": email_id, # PROVENANCE LINK
"past_price": round(past_price, 2),
"purchased_at": history["source_store"],
"purchased_date": history["source_date"],
"purchased_email_id": history["source_id"] # PROVENANCE LINK
}
if past_price > 0 and sale_price <= (past_price * 0.70):
savings_pct = round((1 - sale_price / past_price) * 100)
if name not in alert_map or sale_price < alert_map[name]["sale_price"]:
alert_map[name] = {
"item": name,
"sale_price": round(sale_price, 2),
"deal_store": deal_store,
"deal_date": deal_date,
"deal_email_id": email_id, # PROVENANCE LINK
"past_price": round(past_price, 2),
"purchased_at": history["source_store"],
"purchased_date": history["source_date"],
"purchased_email_id": history["source_id"], # PROVENANCE LINK
"discount": f"{savings_pct}% off",
}
smart_re_ups = list(re_up_map.values())
stock_up_alerts = list(alert_map.values())
meal_plans = []
EXCLUDED = ["coffee", "paper", "towel", "supplement", "protein", "salt", "soap", "detergent", "spray", "cleaning"]
food_items = [i["item"] for i in smart_re_ups if not any(kw in i["item"].lower() for kw in EXCLUDED)]
if food_items:
prompt = (
f"I have these fresh ingredients currently on sale: {', '.join(food_items[:10])}. "
f"Generate exactly 2 gourmet chef-level recipes using primarily these ingredients. "
f"Supplement with common pantry staples as needed. "
f"Each recipe should feel restaurant-quality and use at least 3 listed ingredients."
)
schema = '{"meal_plans": [{"meal_name": "string", "ingredients": ["string"], "steps": ["string"]}]}'
result = extract_json(prompt, schema)
meal_plans = result.get("meal_plans", [])
return {
"grocery_spend_total": round(receipt_total, 2),
"smart_re_ups": smart_re_ups,
"meal_plans": meal_plans,
"stock_up_alerts": stock_up_alerts,
}
def run_groceries(cur: psycopg2.extensions.cursor) -> None:
"""Executes the Groceries & Promotions domain pipeline."""
receipts = get_emails_by_tag(cur, "RECEIPT")
flyers = get_emails_by_tag(cur, "PROMOTION")
r = process_groceries(receipts, flyers)
upsert(cur, "intel_groceries",
["grocery_spend_total", "smart_re_ups", "meal_plans", "stock_up_alerts"],
[r["grocery_spend_total"], r["smart_re_ups"], r["meal_plans"], r["stock_up_alerts"]])
print(f" β³ Spend: ${r['grocery_spend_total']:.2f} | Re-ups: {len(r['smart_re_ups'])} | Alerts: {len(r['stock_up_alerts'])}")
# =============================================================================
# DOMAIN: LOGISTICS
# =============================================================================
def process_logistics(emails: List[Tuple]) -> Dict[str, Any]:
"""Deterministically tracks active shipments and calculates expiring return windows."""
print(" [LOGISTICS] Resolving package states & return policies...")
active = []
blocks = []
returns = []
for email_id, sender, subject, received_at, data in emails:
if not data:
continue
status = data.get("status", "")
if status == "Out for Delivery":
active.append({"vendor": data.get("vendor", sender), "item": data.get("item", "Package"), "status": status, "source_id": email_id})
elif status in ("Customs Hold", "Blocked"):
blocks.append({
"vendor": data.get("vendor", sender),
"status": status,
"fee": data.get("fee", 0),
"action_required": data.get("action_required", f"Resolve {status}"),
"source_id": email_id # PROVENANCE LINK
})
if data.get("return_policy_days") and received_at:
days_since = (datetime.now() - received_at).days
days_left = data["return_policy_days"] - days_since
if 0 <= days_left <= 30:
returns.append({"item": data.get("item", "Item"), "vendor": data.get("vendor", sender), "days_left": days_left, "source_id": email_id})
return {"active_shipments": active, "critical_blocks": blocks, "return_windows": returns}
def run_logistics(cur: psycopg2.extensions.cursor) -> None:
"""Executes the Logistics & Shipping domain pipeline."""
emails = get_emails_by_tag(cur, "LOGISTICS")
r = process_logistics(emails)
upsert(cur, "intel_logistics",
["active_shipments", "critical_blocks", "return_windows"],
[r["active_shipments"], r["critical_blocks"], r["return_windows"]])
print(f" β³ Active: {len(r['active_shipments'])} | Blocked: {len(r['critical_blocks'])} | Expiring Returns: {len(r['return_windows'])}")
# =============================================================================
# DOMAIN: CAREER
# =============================================================================
def process_career(emails: List[Tuple]) -> Dict[str, Any]:
"""Synthesizes job opportunities and required career actions."""
print(" [CAREER] Parsing opportunities and network signals...")
actions = []
opps = []
network = []
for email_id, sender, subject, _, data in emails:
if not data:
continue
if data.get("action_required"):
actions.append({"action": data["action_required"], "contact": data.get("contact", ""), "source_id": email_id})
if data.get("opportunity"):
opps.append({"opportunity": data["opportunity"], "source_id": email_id})
if data.get("network_update"):
network.append({"update": data["network_update"], "source_id": email_id})
return {"action_items": actions, "opportunities": opps, "network_radar": network}
def run_career(cur: psycopg2.extensions.cursor) -> None:
"""Executes the Career & Opportunities domain pipeline."""
emails = get_emails_by_tag(cur, "CAREER")
r = process_career(emails)
upsert(cur, "intel_career",
["action_items", "opportunities", "network_radar"],
[r["action_items"], r["opportunities"], r["network_radar"]])
print(f" β³ Actions: {len(r['action_items'])} | Opps: {len(r['opportunities'])} | Network: {len(r['network_radar'])}")
# =============================================================================
# DOMAIN: NOTIFICATIONS
# =============================================================================
def process_notifications(emails: List[Tuple]) -> Dict[str, Any]:
"""Triages system alerts and collapses low-priority noise into a single digest."""
print(" [ALERTS] Triaging urgent system pings vs. noise...")
urgent = []
system = []
noise_tasks = []
for email_id, sender, subject, _, data in emails:
if not data:
continue
is_urgent = data.get("urgent", False)
source = data.get("source", "")
task = data.get("task", "")
if is_urgent and source == "Datadog":
system.append({"source": source, "task": task, "source_id": email_id})
elif is_urgent:
urgent.append({"source": source, "task": task, "source_id": email_id})
elif task:
noise_tasks.append(task) # Grouped summary, harder to link to 1 email
digest = []
if noise_tasks:
result = extract_json(
f"Compress these low-priority developer tasks into one readable sentence a busy engineer would find useful: {noise_tasks}",
'{"summary": "string"}'
)
if result.get("summary"):
digest.append({"summary": result["summary"]})
return {"urgent_asks": urgent, "system_alerts": system, "read_later_digest": digest}
def run_notifications(cur: psycopg2.extensions.cursor) -> None:
"""Executes the System Notifications domain pipeline."""
emails = get_emails_by_tag(cur, "NOTIFICATION")
r = process_notifications(emails)
upsert(cur, "intel_notifications",
["urgent_asks", "system_alerts", "read_later_digest"],
[r["urgent_asks"], r["system_alerts"], r["read_later_digest"]])
print(f" β³ Urgent: {len(r['urgent_asks'])} | System: {len(r['system_alerts'])} | Digest Items: {len(r['read_later_digest'])}")
# =============================================================================
# DOMAIN: NEWSLETTERS
# =============================================================================
def process_newsletters(emails: List[Tuple]) -> Dict[str, Any]:
"""Identifies industry consensus by mapping cross-newsletter topic frequencies."""
print(" [SIGNALS] Extracting cross-publication consensus...")
topic_map = {}
for email_id, sender, subject, _, data in emails:
if data and data.get("topics"):
for t in data["topics"]:
if t not in topic_map:
topic_map[t] = []
topic_map[t].append(email_id) # PROVENANCE LINK (Array of IDs)
all_topics = list(topic_map.keys())
if not all_topics:
return {"top_headlines": [], "industry_signals": [], "deep_dive": []}
# If it appears in multiple newsletters, it's a signal. Attach all source_ids.
signals = [{"topic": t, "mentions": len(ids), "source_ids": ids} for t, ids in topic_map.items() if len(ids) > 1]
signals = sorted(signals, key=lambda x: -x["mentions"])
llm = extract_json(
f"You are briefing a senior exec at a Canadian fintech. Topics from today's newsletters: {all_topics}. "
f"Select the 3 most strategically important headlines. Identify 1 most technical topic for deep dive. Use exact topic names.",
'{"top_headlines": ["h1","h2","h3"], "deep_dive_topic": "string"}'
)
top_headlines = [{"topic": h, "source_id": topic_map[h][0]} for h in llm.get("top_headlines", []) if h in topic_map]
deep_dive_topic = llm.get("deep_dive_topic", "")
deep_dives = [{"topic": deep_dive_topic, "source_id": topic_map[deep_dive_topic][0]}] if deep_dive_topic in topic_map else []
return {
"top_headlines": top_headlines,
"industry_signals": signals,
"deep_dive": deep_dives,
}
def run_newsletters(cur: psycopg2.extensions.cursor) -> None:
"""Executes the Newsletter & Signal synthesis domain pipeline."""
emails = get_emails_by_tag(cur, "NEWSLETTER")
r = process_newsletters(emails)
upsert(cur, "intel_newsletters",
["top_headlines", "industry_signals", "deep_dive"],
[r["top_headlines"], r["industry_signals"], r["deep_dive"]])
print(f" β³ Headlines: {len(r['top_headlines'])} | Deep Dives: {len(r['deep_dive'])} | Signals: {len(r['industry_signals'])}")
# =============================================================================
# DOMAIN ROUTER
# =============================================================================
DOMAIN_RUNNERS = {
"FINANCE": [run_finance],
"RECEIPT": [run_groceries],
"PROMOTION": [run_groceries],
"LOGISTICS": [run_logistics],
"CAREER": [run_career],
"NOTIFICATION": [run_notifications],
"NEWSLETTER": [run_newsletters],
}
def run_domain(tag: str) -> None:
"""
Targeted execution router. Used to re-process a single domain efficiently
when an end-user manually drags and drops an email in the UI.
"""
runners = DOMAIN_RUNNERS.get(tag.upper(), [])
if not runners:
print(f" [WARN] No runner available for tag: {tag}")
return
conn = get_conn()
cur = conn.cursor()
try:
print(f" [EVENT] Re-processing domain: {tag}")
for runner in runners:
runner(cur)
conn.commit()
print(f" [OK] {tag} intelligence updated successfully.")
except Exception as e:
print(f" [ERR] {tag} processing failed: {e}")
conn.rollback()
raise
finally:
cur.close()
conn.close()
# =============================================================================
# FULL PIPELINE RUN
# =============================================================================
def run_all() -> None:
"""
Master orchestration function. Executes the entire deterministic math
and LLM synthesis pipeline across all domains.
"""
print("\n" + "=" * 65)
print(" [STAGE 4] SYNTHESIS & DETERMINISTIC MATH ENGINE")
print("=" * 65 + "\n")
conn = get_conn()
cur = conn.cursor()
try:
for tag in ["FINANCE", "RECEIPT", "PROMOTION", "LOGISTICS", "CAREER", "NOTIFICATION", "NEWSLETTER"]:
cur.execute("SELECT COUNT(*) FROM emails WHERE librarian_tag = %s", (tag,))
count = cur.fetchone()[0]
print(f" [QUEUED] {tag:<12} -> {count} items")
print()
run_finance(cur); conn.commit()
run_logistics(cur); conn.commit()
run_career(cur); conn.commit()
run_notifications(cur); conn.commit()
run_newsletters(cur); conn.commit()
run_groceries(cur); conn.commit()
print("\n" + "=" * 65)
print(" [OK] Intelligence Lake fully updated with Provenance.")
print("=" * 65 + "\n")
except Exception as e:
print(f"\n [ERR] Orchestration failed: {e}")
import traceback
traceback.print_exc()
conn.rollback()
finally:
cur.close()
conn.close()
# =============================================================================
# CLI ENTRYPOINT
# =============================================================================
if __name__ == "__main__":
if len(sys.argv) > 1:
run_domain(sys.argv[1].upper())
else:
run_all()
05 // Frontend: Provenance & Optimistic Updates
A major issue with AI tools is that users don't trust the summaries. I solved this in the React frontend by implementing Provenance. Every synthesized insight (e.g., a "Smart Re-Up" deal or an "Upcoming Liability") retains its source_id. When a user clicks an AI insight, it instantly surfaces the original source email.
I also implemented Optimistic UI retagging. When a user drags an email to a new category, the UI snaps instantly. To support this without blocking the user, the FastAPI backend delegates the heavy math recalculations to native BackgroundTasks.
import { useState, useEffect } from 'react';
import axios from 'axios';
import {
User, ShieldAlert, TrendingUp, X, ShoppingCart, Tag, Zap, Send,
ArchiveRestore, Archive, Github, Cloud, Chrome, Key, MessageSquare,
Search, ArrowRight, Loader2, PenSquare, Forward, ChevronDown,
Trash2, Paperclip, Download, Receipt, Package, Briefcase,
Bell, Newspaper, BookMarked, GripVertical, MoveRight, History
} from 'lucide-react';
const API_BASE = "http://localhost:8000";
// Tab config
const VAULT_TABS = [
{ id: 'FINANCE', label: 'Finance', icon: TrendingUp, intelKey: 'FINANCE' },
{ id: 'RECEIPT', label: 'Receipts', icon: Receipt, intelKey: 'GROCERIES' },
{ id: 'PROMOTION', label: 'Deals', icon: Tag, intelKey: 'GROCERIES' },
{ id: 'LOGISTICS', label: 'Logistics', icon: Package, intelKey: 'LOGISTICS' },
{ id: 'CAREER', label: 'Career', icon: Briefcase, intelKey: 'CAREER' },
{ id: 'NOTIFICATION', label: 'Alerts', icon: Bell, intelKey: 'NOTIFICATION' },
{ id: 'NEWSLETTER', label: 'Newsletters', icon: Newspaper, intelKey: 'NEWSLETTER' },
{ id: 'ARCHIVED', label: 'Archived', icon: BookMarked, intelKey: null },
];
// π’ NEW: Extended menu config so we can move items back to the Home Screen
const MOVE_MENU_TABS = [
{ id: 'HUMAN', label: 'Direct Messages', icon: User },
{ id: 'KEY', label: 'Auth Events', icon: ShieldAlert },
...VAULT_TABS
];
const INTEL_META: Record<string, { title: string; subtitle: string }> = {
FINANCE: { title: 'Finance Intelligence', subtitle: 'Upcoming bills and liabilities.' },
RECEIPT: { title: 'Spend Intelligence', subtitle: 'Chronological breakdown of your spending habits.' },
PROMOTION: { title: 'Deal Intelligence', subtitle: 'Active sales cross-referenced against your purchase history.' },
LOGISTICS: { title: 'Delivery Intelligence',subtitle: 'Active shipments, customs blocks, and expiring returns.' },
CAREER: { title: 'Career Intelligence', subtitle: 'Actions required, opportunities, and network signals.' },
NOTIFICATION: { title: 'Alert Intelligence', subtitle: 'Urgent asks, system fires, and read-later digest.' },
NEWSLETTER: { title: 'Signal Intelligence', subtitle: 'Top headlines, trending topics, and deep dives.' },
};
const getLogo = (sender: string) => {
const domain = sender.includes('@') ? sender.split('@')[1] : sender;
if (!domain) return null;
return `https://www.google.com/s2/favicons?domain=${domain}&sz=128`;
};
const getAuthIcon = (sender: string, subject: string) => {
const text = (sender + subject).toLowerCase();
if (text.includes('github')) return <Github size={16} className="text-stone-700" />;
if (text.includes('aws') || text.includes('amazon')) return <Cloud size={16} className="text-amber-500" />;
if (text.includes('google') || text.includes('okta')) return <Chrome size={16} className="text-emerald-500" />;
if (text.includes('slack')) return <MessageSquare size={16} className="text-indigo-500" />;
if (text.includes('wealthsimple')) return <span className="text-[10px] font-black text-emerald-600">WS</span>;
return <Key size={16} className="text-orange-500" />;
};
const formatShortDate = (dateString: string) => {
if (!dateString) return '';
const date = new Date(dateString);
const now = new Date();
if (date.toDateString() === now.toDateString()) return date.toLocaleTimeString('en-US', { hour: 'numeric', minute: '2-digit' });
return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
};
const formatLongDate = (dateString: string) => {
if (!dateString) return '';
return new Date(dateString).toLocaleDateString('en-US', {
weekday: 'long', year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: '2-digit'
});
};
const groupFeedByDate = (emails: any[]) => {
const groups: Record<string, any[]> = { 'Today': [], 'This Week': [], 'Older': [] };
const now = new Date();
emails.forEach(e => {
if (!e.received_at) return;
const d = new Date(e.received_at);
const diffDays = Math.floor((now.getTime() - d.getTime()) / 86400000);
if (diffDays < 1) groups['Today'].push(e);
else if (diffDays <= 7) groups['This Week'].push(e);
else groups['Older'].push(e);
});
return groups;
};
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// INTELLIGENCE PANEL
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
const IntelPanel = ({ activeTab, intel, feed, setSelectedEmail, archivedIds }: any) => {
const meta = INTEL_META[activeTab];
if (activeTab === 'ARCHIVED') return null;
// ββ FINANCE: Upcoming Liabilities Only βββββββββββββββββββββββββββββ
if (activeTab === 'FINANCE') {
const liabs = intel?.FINANCE?.upcoming_liabilities || [];
return (
<div className="space-y-6">
<div>
<h2 className="text-4xl font-serif font-black italic leading-none tracking-tighter mb-1">{meta.title}</h2>
<p className="text-[11px] font-bold text-stone-400 uppercase tracking-widest">{meta.subtitle}</p>
</div>
{liabs.length > 0 ? (
<div className="bg-white/60 p-6 rounded-[2rem] border border-stone-100 shadow-sm">
<h4 className="text-[10px] font-black tracking-[0.3em] text-red-500 mb-4 uppercase">Upcoming Liabilities</h4>
<div className="space-y-3">
{liabs.map((item: any, i: number) => {
const isClickable = !!item.source_id;
return (
<div
key={i}
onClick={(ev) => {
if (!isClickable) return;
ev.stopPropagation();
const src = feed.find((e: any) => Number(e.id) === Number(item.source_id));
if (src) setSelectedEmail(src);
}}
className={`flex justify-between items-center py-3 border-b border-stone-50 last:border-0 ${isClickable ? 'cursor-pointer hover:bg-stone-50 transition-colors px-2 -mx-2 rounded-xl' : ''}`}
>
<div className="flex flex-col">
<span className="text-sm font-bold text-stone-800">{item.vendor}</span>
{item.due_date && <span className="text-[10px] font-black text-stone-400 uppercase tracking-widest mt-0.5">DUE: {item.due_date}</span>}
</div>
<span className="text-lg font-black text-red-600">${Number(item.upcoming_charge).toFixed(2)}</span>
</div>
);
})}
</div>
</div>
) : (
<p className="text-stone-400 italic text-sm">No upcoming liabilities detected.</p>
)}
</div>
);
}
// ββ RECEIPT: Chronological Spend Intelligence ββββββββββββββββββββββββββββββ
if (activeTab === 'RECEIPT') {
const receipts = feed.filter((e: any) => e.librarian_tag === 'RECEIPT' && !archivedIds.has(e.id));
const sortedReceipts = [...receipts].sort((a, b) => new Date(b.received_at).getTime() - new Date(a.received_at).getTime());
return (
<div className="space-y-6">
<div>
<h2 className="text-4xl font-serif font-black italic leading-none tracking-tighter mb-1">{meta.title}</h2>
<p className="text-[11px] font-bold text-stone-400 uppercase tracking-widest">{meta.subtitle}</p>
</div>
<div className="space-y-4">
{sortedReceipts.map(receipt => {
const data = receipt.extracted_data;
if (!data) return null;
return (
<div key={receipt.id} onClick={() => setSelectedEmail(receipt)} className="bg-white/60 p-5 rounded-[2rem] border border-stone-100 shadow-sm cursor-pointer hover:border-emerald-300 hover:bg-emerald-50/20 transition-all">
<div className="flex justify-between items-center mb-3">
<div className="flex flex-col">
<span className="font-black text-stone-800 text-base">{data.vendor || receipt.sender.split('@')[0]}</span>
<span className="text-[10px] font-bold text-stone-400 uppercase">{formatShortDate(receipt.received_at)}</span>
</div>
<span className="font-black text-emerald-600 text-lg">${data.total?.toFixed(2)}</span>
</div>
<div className="space-y-1">
{data.items?.slice(0, 3).map((item: any, i: number) => (
<div key={i} className="flex justify-between text-xs font-medium text-stone-600">
<span className="truncate pr-4">{item.item_name}</span>
<span>${item.price?.toFixed(2) || item.sale_price?.toFixed(2)}</span>
</div>
))}
{data.items?.length > 3 && (
<div className="text-[10px] font-bold text-stone-400 pt-1">+{data.items.length - 3} more items</div>
)}
</div>
</div>
)
})}
</div>
{sortedReceipts.length === 0 && (
<p className="text-stone-400 italic text-sm">Run the pipeline to generate spend intelligence.</p>
)}
</div>
);
}
// ββ PROMOTION: Deal Intelligence with Hyperlinks βββββββββββββββββββββββββββββ
if (activeTab === 'PROMOTION') {
const groceries = intel?.GROCERIES;
if (!groceries) return (
<div className="space-y-4">
<div>
<h2 className="text-4xl font-serif font-black italic leading-none tracking-tighter mb-1">{meta.title}</h2>
<p className="text-[11px] font-bold text-stone-400 uppercase tracking-widest">{meta.subtitle}</p>
</div>
<p className="text-stone-400 italic text-sm">Run the pipeline to generate deal intelligence.</p>
</div>
);
const reups: any[] = groceries.smart_re_ups || [];
const alerts: any[] = groceries.stock_up_alerts || [];
const meals: any[] = groceries.meal_plans || [];
const getIngredientSource = (ing: string) => {
let match = alerts.find(a => a.item.toLowerCase().includes(ing.toLowerCase()) || ing.toLowerCase().includes(a.item.toLowerCase()));
if (match) return match.deal_email_id;
match = reups.find(r => r.item.toLowerCase().includes(ing.toLowerCase()) || ing.toLowerCase().includes(r.item.toLowerCase()));
if (match) return match.deal_email_id;
return null;
};
return (
<div className="space-y-6">
<div>
<h2 className="text-4xl font-serif font-black italic leading-none tracking-tighter mb-1">{meta.title}</h2>
<p className="text-[11px] font-bold text-stone-400 uppercase tracking-widest">{meta.subtitle}</p>
</div>
{/* Stock-up alerts */}
{alerts.length > 0 && (
<div className="bg-orange-50 border border-orange-100 rounded-[2rem] p-6 shadow-sm">
<h4 className="text-[10px] font-black tracking-[0.3em] text-orange-500 mb-4 uppercase">Stock-Up Alerts - 30%+ Off</h4>
<div className="space-y-4">
{alerts.map((item: any, i: number) => (
<div key={i} className="py-3 border-b border-orange-100 last:border-0">
<div className="flex items-center justify-between mb-2">
<span className="text-sm font-black text-stone-900">{item.item}</span>
<div className="flex items-center gap-2">
<span className="text-xs text-stone-400 line-through">${item.past_price?.toFixed(2)}</span>
<span className="text-[10px] font-black bg-orange-500 text-white px-2 py-0.5 rounded-md">{item.discount}</span>
<span className="text-sm font-black text-emerald-700">${item.sale_price?.toFixed(2)}</span>
</div>
</div>
<div className="flex items-center gap-2 flex-wrap mt-1">
<button
onClick={(ev) => {
ev.stopPropagation();
if (!item.deal_email_id) return alert("Missing ID! Run: python intelligence_worker.py GROCERIES");
const src = feed.find((e: any) => Number(e.id) === Number(item.deal_email_id));
if (src) setSelectedEmail(src);
}}
className="text-[10px] font-bold text-orange-700 bg-orange-200/50 px-3 py-1.5 rounded-md cursor-pointer hover:bg-orange-200 transition-colors flex items-center gap-1.5 shadow-sm"
>
<Tag size={10} /> Deal at {item.deal_store}{item.deal_date ? ` Β· ${item.deal_date}` : ''}
</button>
<button
onClick={(ev) => {
ev.stopPropagation();
if (!item.purchased_email_id) return alert("Missing ID! Run: python intelligence_worker.py GROCERIES");
const src = feed.find((e: any) => Number(e.id) === Number(item.purchased_email_id));
if (src) setSelectedEmail(src);
}}
className="text-[10px] font-bold text-stone-500 bg-white/60 border border-stone-200 px-3 py-1.5 rounded-md cursor-pointer hover:bg-stone-100 transition-colors flex items-center gap-1.5 shadow-sm"
>
<Receipt size={10} /> Paid ${item.past_price?.toFixed(2)} at {item.purchased_at}{item.purchased_date ? ` on ${item.purchased_date}` : ''}
</button>
</div>
</div>
))}
</div>
</div>
)}
{/* Smart re-ups */}
{reups.length > 0 && (
<div className="bg-white/60 p-6 rounded-[2rem] border border-stone-100 shadow-sm">
<h4 className="text-[10px] font-black tracking-[0.3em] text-emerald-600 mb-4 uppercase">Smart Re-Ups</h4>
<div className="space-y-4">
{reups.map((item: any, i: number) => (
<div key={i} className="py-3 border-b border-stone-50 last:border-0">
<div className="flex items-center justify-between mb-2">
<span className="text-sm font-black text-stone-900">{item.item}</span>
<div className="flex items-center gap-2">
<span className="text-xs text-stone-400 line-through">${item.past_price?.toFixed(2)}</span>
<span className="text-sm font-black text-emerald-600">${item.sale_price?.toFixed(2)}</span>
</div>
</div>
<div className="flex items-center gap-2 flex-wrap">
<button
onClick={(ev) => {
ev.stopPropagation();
if (!item.deal_email_id) return alert("Missing ID! Run: python intelligence_worker.py GROCERIES");
const src = feed.find((e: any) => Number(e.id) === Number(item.deal_email_id));
if (src) setSelectedEmail(src);
}}
className="text-[10px] font-bold text-emerald-700 bg-emerald-100/50 px-3 py-1.5 rounded-md cursor-pointer hover:bg-emerald-100 transition-colors flex items-center gap-1.5 shadow-sm"
>
<Tag size={10} /> Sale at {item.deal_store}{item.deal_date ? ` (${item.deal_date})` : ''}
</button>
<button
onClick={(ev) => {
ev.stopPropagation();
if (!item.purchased_email_id) return alert("Missing ID! Run: python intelligence_worker.py GROCERIES");
const src = feed.find((e: any) => Number(e.id) === Number(item.purchased_email_id));
if (src) setSelectedEmail(src);
}}
className="text-[10px] font-bold text-stone-500 bg-white/60 border border-stone-200 px-3 py-1.5 rounded-md cursor-pointer hover:bg-stone-100 transition-colors flex items-center gap-1.5 shadow-sm"
>
<Receipt size={10} /> Last bought at {item.purchased_at}{item.purchased_date ? ` on ${item.purchased_date}` : ''}
</button>
</div>
</div>
))}
</div>
</div>
)}
{/* Meal plans */}
{meals.length > 0 && (
<div className="bg-white/60 p-6 rounded-[2rem] border border-stone-100 shadow-sm">
<h4 className="text-[10px] font-black tracking-[0.3em] text-emerald-600 mb-4 uppercase">AI Meal Plans</h4>
<div className="space-y-4">
{meals.map((meal: any, i: number) => (
<div key={i} className="bg-white p-5 rounded-2xl border border-stone-50 shadow-sm">
<p className="text-base font-serif font-black italic text-stone-900 mb-2">{meal.meal_name}</p>
<div className="flex flex-wrap gap-1 mb-3">
{meal.ingredients?.map((ing: string) => {
const sourceId = getIngredientSource(ing);
return (
<button
key={ing}
onClick={(ev) => {
ev.stopPropagation();
if (!sourceId) return;
const src = feed.find((e: any) => Number(e.id) === Number(sourceId));
if (src) setSelectedEmail(src);
}}
className={`text-[9px] font-bold border px-2 py-1 rounded-full transition-colors ${
sourceId
? 'border-emerald-300 bg-emerald-50 text-emerald-700 cursor-pointer hover:bg-emerald-200 hover:border-emerald-400 shadow-sm'
: 'border-stone-200 text-stone-500 bg-stone-50 cursor-default'
}`}
>
{ing} {sourceId && <Zap size={8} className="inline mb-0.5" />}
</button>
);
})}
</div>
{meal.steps?.slice(0, 3).map((step: string, si: number) => (
<p key={si} className="text-xs text-stone-500 leading-relaxed mt-1">β’ {step}</p>
))}
</div>
))}
</div>
</div>
)}
</div>
);
}
// ββ ALL OTHER TABS β Generic structured renderer with Provenance ββββββββββββββ
const rawData = intel?.[activeTab];
if (!rawData) return (
<div className="space-y-4">
<div>
<h2 className="text-4xl font-serif font-black italic leading-none tracking-tighter mb-1">{meta?.title}</h2>
<p className="text-[11px] font-bold text-stone-400 uppercase tracking-widest">{meta?.subtitle}</p>
</div>
<p className="text-stone-400 italic text-sm">Run the pipeline to generate intelligence.</p>
</div>
);
return (
<div className="space-y-6">
<div>
<h2 className="text-4xl font-serif font-black italic leading-none tracking-tighter mb-1">{meta?.title}</h2>
<p className="text-[11px] font-bold text-stone-400 uppercase tracking-widest">{meta?.subtitle}</p>
</div>
{Object.entries(rawData).map(([key, value]: [string, any], idx) => {
if (key === 'report_date' || value === null || value === undefined) return null;
const title = key.replace(/_/g, ' ').toUpperCase();
return (
<div key={idx} className="bg-white/60 p-6 rounded-[2rem] border border-stone-100 shadow-sm">
<h4 className="text-[10px] font-black tracking-[0.3em] text-emerald-600 mb-4 border-b border-emerald-50 pb-2 uppercase">{title}</h4>
{Array.isArray(value) ? (
value.length > 0 ? (
<div className="space-y-3">
{value.map((item: any, i: number) => {
// π’ UNIVERSAL PROVENANCE LOGIC
const targetId = item.source_id || (item.source_ids && item.source_ids[0]);
const isClickable = !!targetId;
return (
<div
key={i}
onClick={(ev) => {
if (!isClickable) return;
ev.stopPropagation();
const src = feed.find((e: any) => Number(e.id) === Number(targetId));
if (src) setSelectedEmail(src);
}}
className={`flex items-start gap-3 py-2 border-b border-stone-50 last:border-0 ${isClickable ? 'cursor-pointer hover:bg-stone-50 transition-colors px-2 -mx-2 rounded-xl' : ''}`}
>
<Zap size={14} className="mt-0.5 shrink-0 text-emerald-400" />
<div className="flex-1 text-sm font-bold text-stone-800 leading-snug">
{item.status && item.vendor && (
<div className="flex justify-between">
<span>{item.vendor} β {item.item || item.status}</span>
<span className={`text-[10px] font-black px-2 py-0.5 rounded-md ${item.status === 'Customs Hold' ? 'bg-red-100 text-red-600' : 'bg-emerald-100 text-emerald-700'}`}>{item.status}</span>
</div>
)}
{item.days_left !== undefined && (
<div className="flex justify-between">
<span>{item.item} ({item.vendor})</span>
<span className="text-[10px] font-black bg-red-100 text-red-600 px-2 py-0.5 rounded-md">{item.days_left}d left</span>
</div>
)}
{(item.action_required || item.action) && !item.vendor && !item.status && (
<span>{item.action_required || item.action}</span>
)}
{item.opportunity && <span>{item.opportunity}</span>}
{(item.network_update || item.update) && !item.vendor && <span>{item.network_update || item.update}</span>}
{item.task && !item.action_required && !item.action && <span>{item.task}</span>}
{item.summary && <span>{item.summary}</span>}
{item.topic && item.mentions !== undefined && (
<div className="flex justify-between">
<span>{item.topic}</span>
<span className="text-[10px] font-black bg-stone-100 text-stone-500 px-2 py-0.5 rounded-md hover:bg-stone-200 transition-colors">{item.mentions}x Sources</span>
</div>
)}
{item.topic && item.mentions === undefined && (
<div className="flex items-center gap-2">
<span className="text-[10px] font-black bg-emerald-100 text-emerald-700 px-2 py-0.5 rounded-md uppercase tracking-widest">Deep Dive</span>
<span>{item.topic}</span>
</div>
)}
{!item.vendor && !item.status && !item.days_left && !item.action_required && !item.action && !item.opportunity && !item.network_update && !item.update && !item.task && !item.summary && !item.topic && (
<span>{typeof item === 'object' ? JSON.stringify(item) : item}</span>
)}
</div>
</div>
);
})}
</div>
) : (
<p className="text-sm text-stone-400 italic">None detected.</p>
)
) : (
<div className="bg-white px-5 py-3 rounded-2xl border border-stone-200 shadow-sm inline-flex items-center gap-3">
<span className="font-black text-stone-400 uppercase tracking-widest text-[10px]">Total:</span>
<span className="font-black text-2xl text-emerald-600">
{typeof value === 'number' ? `$${value.toFixed(2)}` : String(value)}
</span>
</div>
)}
</div>
);
})}
</div>
);
};
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
// MAIN APP
// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
export default function App() {
const [intel, setIntel] = useState<any>(null);
const [feed, setFeed] = useState<any[]>([]);
const [view, setView] = useState<'home' | 'vault' | 'sent'>('home');
const [activeTab, setActiveTab] = useState('FINANCE');
const [sentEmails, setSentEmails] = useState<any[]>([]);
const [searchQuery, setSearchQuery] = useState('');
const [isSearching, setIsSearching] = useState(false);
const [searchResults, setSearchResults] = useState<any[] | null>(null);
const [archivedIds, setArchivedIds] = useState<Set<number>>(new Set());
const [selectedEmail, setSelectedEmail] = useState<any>(null);
const [replyText, setReplyText] = useState('');
const [showCC, setShowCC] = useState(false);
const [ccText, setCcText] = useState('');
const [bccText, setBccText] = useState('');
const [isSending, setIsSending] = useState(false);
const [isComposing, setIsComposing] = useState(false);
const [composeData, setComposeData] = useState({ to: '', cc: '', bcc: '', subject: '', body: '' });
const [retagMenuId, setRetagMenuId] = useState<number | null>(null);
const [draggedEmail, setDraggedEmail] = useState<any>(null);
const [dragOverTab, setDragOverTab] = useState<string | null>(null);
const [isRetagging, setIsRetagging] = useState<number | null>(null);
const fetchDashboardData = async () => {
try {
const [intelRes, feedRes] = await Promise.all([
axios.get(`${API_BASE}/api/intelligence`),
axios.get(`${API_BASE}/api/live-feed`),
]);
setIntel(intelRes.data);
setFeed(feedRes.data);
} catch (e) { console.error('Database Fetch Failed'); }
};
useEffect(() => {
fetchDashboardData();
const interval = setInterval(fetchDashboardData, 30000);
return () => clearInterval(interval);
}, []);
useEffect(() => {
const handler = () => setRetagMenuId(null);
document.addEventListener('click', handler);
return () => document.removeEventListener('click', handler);
}, []);
const toggleArchive = (id: number) => {
setArchivedIds(prev => {
const next = new Set(prev);
next.has(id) ? next.delete(id) : next.add(id);
return next;
});
if (selectedEmail?.id === id) setSelectedEmail(null);
};
const handleDelete = async (id: number) => {
try {
await axios.delete(`${API_BASE}/api/email/${id}`);
setFeed(prev => prev.filter(e => e.id !== id));
if (selectedEmail?.id === id) setSelectedEmail(null);
} catch (e) { console.error('Delete failed'); }
};
const handleSearch = async (e: React.FormEvent) => {
e.preventDefault();
if (!searchQuery.trim()) { setSearchResults(null); return; }
setIsSearching(true);
try {
const res = await axios.get(`${API_BASE}/api/search?q=${encodeURIComponent(searchQuery)}`);
setSearchResults(res.data);
} catch (e) { console.error('Search failed'); } finally { setIsSearching(false); }
};
const handleRetag = async (emailId: number, newTag: string, oldTag: string) => {
if (newTag === oldTag) return;
setRetagMenuId(null);
setIsRetagging(emailId);
try {
// 1. Send the update to the backend (FastAPI BackgroundTasks handles the math silently)
await axios.patch(`${API_BASE}/api/email/${emailId}/retag`, {
new_tag: newTag,
old_tag: oldTag,
});
// 2. OPTIMISTIC UI (Feed): Instantly route the item to its new tab / view
setFeed(prev => prev.map(e => {
if (e.id === emailId) {
return {
...e,
librarian_tag: newTag,
is_human: newTag === 'HUMAN',
is_key: newTag === 'KEY'
};
}
return e;
}));
// 3. OPTIMISTIC UI (Intelligence): Instantly snap the item out of the math panels
setIntel((prev: any) => {
if (!prev) return prev;
const next = JSON.parse(JSON.stringify(prev)); // Deep copy to avoid state mutation
// Strip from Finance Liabilities
if (oldTag === 'FINANCE' && next.FINANCE?.upcoming_liabilities) {
next.FINANCE.upcoming_liabilities = next.FINANCE.upcoming_liabilities.filter(
(item: any) => Number(item.source_id) !== Number(emailId)
);
}
// Strip from Logistics Shipments
if (oldTag === 'LOGISTICS' && next.LOGISTICS?.active_shipments) {
next.LOGISTICS.active_shipments = next.LOGISTICS.active_shipments.filter(
(item: any) => Number(item.source_id) !== Number(emailId)
);
}
return next;
});
} catch (e) {
console.error('Retag failed', e);
} finally {
setIsRetagging(null);
}
};
const handleSendReply = async (isForward = false) => {
if ((!replyText.trim() && !isForward) || (isForward && !ccText.includes('@'))) return;
setIsSending(true);
try {
await axios.post(`${API_BASE}/api/send`, {
to_address: isForward ? ccText : selectedEmail.sender,
cc_address: isForward ? '' : ccText,
bcc_address: bccText,
subject: isForward ? `Fwd: ${selectedEmail.subject}` : `Re: ${selectedEmail.subject}`,
body: isForward ? `\n\n--- Forwarded Message ---\n${selectedEmail.body_text}` : replyText,
reply_to_id: selectedEmail.id, is_forward: isForward,
});
const newSentEmail = {
id: Date.now(),
sender: `To: ${isForward ? ccText : selectedEmail.sender}`,
subject: isForward ? `Fwd: ${selectedEmail.subject}` : `Re: ${selectedEmail.subject}`,
body_text: isForward ? `\n\n--- Forwarded Message ---\n${selectedEmail.body_text}` : replyText,
received_at: new Date().toISOString(),
is_human: true,
librarian_tag: 'SENT'
};
setSentEmails(prev => [newSentEmail, ...prev]);
setReplyText(''); setCcText(''); setBccText(''); setShowCC(false); setSelectedEmail(null);
} catch (e) { console.error('Send failed'); } finally { setIsSending(false); }
};
const handleSendNew = async () => {
if (!composeData.to || !composeData.body) return;
setIsSending(true);
try {
await axios.post(`${API_BASE}/api/send`, {
to_address: composeData.to, cc_address: composeData.cc,
bcc_address: composeData.bcc, subject: composeData.subject, body: composeData.body,
});
const newSentEmail = {
id: Date.now(),
sender: `To: ${composeData.to}`,
subject: composeData.subject,
body_text: composeData.body,
received_at: new Date().toISOString(),
is_human: true,
librarian_tag: 'SENT'
};
setSentEmails(prev => [newSentEmail, ...prev]);
setIsComposing(false);
setComposeData({ to: '', cc: '', bcc: '', subject: '', body: '' });
} catch (e) { console.error('Send failed'); } finally { setIsSending(false); }
};
let displayFeed: any[] = [];
if (activeTab === 'ARCHIVED') {
// Show items archived via the button OR via drag-and-drop
displayFeed = feed.filter(e => archivedIds.has(e.id) || e.librarian_tag === 'ARCHIVED');
} else {
// Hide items if they are archived
displayFeed = feed.filter(e => e.librarian_tag === activeTab && !archivedIds.has(e.id) && e.librarian_tag !== 'ARCHIVED');
}
const groupedFeed = groupFeedByDate(displayFeed);
return (
<div className="min-h-screen bg-[#FDFCF8] text-stone-900 font-sans flex flex-col relative">
{/* ββ COMPOSE MODAL βββββββββββββββββββββββββββββββββββββββ */}
{isComposing && (
<div className="fixed inset-0 z-[200] bg-stone-900/60 backdrop-blur-md flex items-center justify-center p-8">
<div className="bg-white w-full max-w-4xl rounded-[3rem] shadow-2xl overflow-hidden border-2 border-stone-900 flex flex-col h-[70vh] animate-in zoom-in-95 duration-200">
<div className="flex justify-between items-center p-6 border-b border-stone-100 bg-stone-50">
<div className="flex items-center gap-2 text-stone-800 font-black tracking-widest uppercase text-sm">
<PenSquare size={16} /> New Message
</div>
<button onClick={() => setIsComposing(false)} className="hover:bg-stone-200 p-2 rounded-full transition-colors"><X size={20} /></button>
</div>
<div className="flex flex-col flex-1 overflow-y-auto">
<input type="text" placeholder="To" value={composeData.to} onChange={e => setComposeData({ ...composeData, to: e.target.value })} className="w-full border-b border-stone-100 p-4 outline-none font-medium text-stone-800" />
{showCC && (
<>
<input type="text" placeholder="Cc" value={composeData.cc} onChange={e => setComposeData({ ...composeData, cc: e.target.value })} className="w-full border-b border-stone-100 p-4 outline-none font-medium text-stone-800" />
<input type="text" placeholder="Bcc" value={composeData.bcc} onChange={e => setComposeData({ ...composeData, bcc: e.target.value })} className="w-full border-b border-stone-100 p-4 outline-none font-medium text-stone-800" />
</>
)}
<input type="text" placeholder="Subject" value={composeData.subject} onChange={e => setComposeData({ ...composeData, subject: e.target.value })} className="w-full border-b border-stone-100 p-4 outline-none font-bold text-stone-800 text-lg" />
<textarea placeholder="Write something..." value={composeData.body} onChange={e => setComposeData({ ...composeData, body: e.target.value })} className="w-full flex-1 p-6 outline-none resize-none font-medium text-stone-700" />
</div>
<div className="p-6 bg-stone-50 border-t border-stone-100 flex justify-between items-center">
<button onClick={() => setShowCC(!showCC)} className="text-[10px] font-black uppercase tracking-widest text-stone-400 hover:text-stone-800 transition-colors flex items-center gap-1">
CC / BCC <ChevronDown size={12} className={showCC ? 'rotate-180' : ''} />
</button>
<button onClick={handleSendNew} disabled={isSending} className="bg-stone-900 text-white px-8 py-3 rounded-full text-[10px] font-black uppercase tracking-widest hover:bg-emerald-600 flex items-center gap-2 transition-colors disabled:opacity-50">
{isSending ? 'Sending...' : 'Send'} <Send size={12} />
</button>
</div>
</div>
</div>
)}
{/* ββ EMAIL DETAIL MODAL ββββββββββββββββββββββββββββββββββ */}
{selectedEmail && (() => {
const hasShoppingList = selectedEmail.extracted_data && selectedEmail.extracted_data.items && selectedEmail.extracted_data.items.length > 0;
const hasSidebar = selectedEmail.file_name || hasShoppingList;
return (
<div className="fixed inset-0 z-[100] bg-stone-900/60 backdrop-blur-md flex items-center justify-center p-8">
<div className="bg-white w-full max-w-6xl rounded-[4rem] shadow-2xl overflow-hidden border-2 border-stone-900 flex h-[85vh] relative animate-in zoom-in-95 duration-200">
<div className="absolute top-8 right-8 flex gap-3 z-10">
<button onClick={() => handleDelete(selectedEmail.id)} className="bg-white/80 backdrop-blur hover:bg-red-100 hover:text-red-600 text-stone-400 p-4 rounded-full transition-colors border border-stone-100 shadow-sm" title="Permanently Delete">
<Trash2 size={20}/>
</button>
<div className="relative">
<button
onClick={(ev) => { ev.stopPropagation(); setRetagMenuId(retagMenuId === selectedEmail.id ? null : selectedEmail.id); }}
className="bg-white/80 backdrop-blur hover:bg-emerald-100 hover:text-emerald-600 text-stone-400 p-4 rounded-full transition-colors border border-stone-100 shadow-sm"
title="Move to..."
>
<MoveRight size={20}/>
</button>
{retagMenuId === selectedEmail.id && (
<div className="absolute right-0 top-16 bg-white border-2 border-stone-900 rounded-2xl shadow-xl z-50 overflow-hidden min-w-[180px]">
<p className="text-[9px] font-black uppercase tracking-widest text-stone-400 px-4 pt-3 pb-1">File as Intelligence</p>
{/* π’ MAP OVER MOVE_MENU_TABS INSTEAD OF VAULT_TABS */}
{MOVE_MENU_TABS.filter(t => t.id !== selectedEmail.librarian_tag).map(t => {
const TIcon = t.icon;
return (
<button
key={t.id}
onClick={() => handleRetag(selectedEmail.id, t.id, selectedEmail.librarian_tag)}
className="w-full flex items-center gap-3 px-4 py-3 text-sm font-bold text-stone-700 hover:bg-stone-50 transition-colors border-b last:border-0 border-stone-50"
>
<TIcon size={14} className={t.id === 'ARCHIVED' ? "text-stone-400" : "text-emerald-500"} /> {t.label}
</button>
);
})}
</div>
)}
</div>
<button onClick={() => toggleArchive(selectedEmail.id)} className="bg-white/80 backdrop-blur hover:bg-emerald-100 hover:text-emerald-600 text-stone-400 p-4 rounded-full transition-colors border border-stone-100 shadow-sm" title={archivedIds.has(selectedEmail.id) ? "Unarchive" : "Archive"}>
{archivedIds.has(selectedEmail.id) ? <ArchiveRestore size={20}/> : <Archive size={20}/>}
</button>
<button onClick={() => {setSelectedEmail(null); setRetagMenuId(null);}} className="bg-white/80 backdrop-blur hover:bg-stone-200 p-4 rounded-full transition-colors border border-stone-100 shadow-sm">
<X size={20}/>
</button>
</div>
<div className={`flex flex-col bg-white ${hasSidebar ? 'flex-1 border-r border-stone-100' : 'w-full'}`}>
<div className="p-16 overflow-y-auto flex-1 pt-24">
<h3 className="text-4xl font-serif font-black italic mb-6 leading-tight pr-40">{selectedEmail.subject}</h3>
<div className="flex items-center gap-4 mb-12">
<p className="text-[11px] font-black text-emerald-600 bg-emerald-50 px-6 py-2 rounded-full uppercase tracking-widest">{selectedEmail.sender}</p>
<p className="text-[11px] font-bold text-stone-400 uppercase tracking-widest">{formatLongDate(selectedEmail.received_at)}</p>
</div>
<div className="text-stone-600 leading-relaxed font-medium text-lg whitespace-pre-wrap italic">{selectedEmail.body_text}</div>
</div>
{/* π’ REMOVED selectedEmail.is_human SO YOU CAN REPLY TO ANYTHING */}
{view !== 'sent' && (
<div className="p-8 bg-stone-50 border-t border-stone-100">
<div className="bg-white border-2 border-stone-200 focus-within:border-stone-900 transition-colors rounded-[2rem] p-4 flex flex-col shadow-sm">
{(showCC || replyText.startsWith('Fwd:')) && (
<div className="flex gap-4 mb-2 border-b border-stone-50 pb-2 px-2">
<input type="text" placeholder="Forward to address (e.g., manager@company.com)" value={ccText} onChange={e => setCcText(e.target.value)} className="flex-1 outline-none text-xs font-bold text-emerald-600 placeholder:text-stone-300" />
</div>
)}
<textarea
value={replyText}
onChange={e => setReplyText(e.target.value)}
placeholder={`Draft a secure response to ${selectedEmail.sender.split('@')[0]}...`}
className="w-full bg-transparent resize-none outline-none text-sm font-medium text-stone-800 p-2 min-h-[80px]"
/>
<div className="flex justify-between items-center mt-2 px-2">
<button onClick={() => setShowCC(!showCC)} className="text-[9px] font-black uppercase tracking-widest text-stone-400 hover:text-stone-800 flex items-center gap-1">
CC / BCC
</button>
<div className="flex gap-2">
<button
onClick={() => handleSendReply(true)}
disabled={isSending || !ccText.includes('@')}
className="bg-stone-100 text-stone-600 px-6 py-2.5 rounded-full text-[10px] font-black uppercase tracking-widest hover:bg-stone-200 flex items-center gap-2 transition-colors disabled:opacity-50"
>
Forward <Forward size={12} />
</button>
<button
onClick={() => handleSendReply(false)}
disabled={isSending || !replyText.trim()}
className="bg-stone-900 text-white px-6 py-2.5 rounded-full text-[10px] font-black uppercase tracking-widest hover:bg-emerald-600 flex items-center gap-2 transition-colors disabled:opacity-50"
>
{isSending ? 'Sending...' : 'Send Reply'} <Send size={12} />
</button>
</div>
</div>
</div>
</div>
)}
</div>
{hasSidebar && (
<div className="w-[420px] bg-stone-50/50 p-10 pt-28 flex flex-col overflow-y-auto gap-6">
{selectedEmail.file_name && (
<div className="bg-white p-8 rounded-[3rem] border-2 border-stone-900 shadow-xl flex flex-col">
<div className="flex items-center gap-4 mb-6 text-emerald-600">
<Paperclip size={20} />
<span className="font-black text-sm uppercase tracking-widest">Attachment</span>
</div>
<div className="flex items-center justify-between p-4 bg-stone-50 rounded-3xl border border-stone-100 mb-4">
<span className="text-xs font-bold text-stone-600 truncate mr-4">{selectedEmail.file_name}</span>
<a href={`${API_BASE}/attachments/${selectedEmail.file_hash}.pdf`} target="_blank" rel="noreferrer" className="bg-stone-900 text-white p-3 rounded-full hover:bg-emerald-600 transition-colors shrink-0">
<Download size={14} />
</a>
</div>
<iframe src={`${API_BASE}/attachments/${selectedEmail.file_hash}.pdf#toolbar=0`} className="w-full h-[400px] rounded-2xl border border-stone-100 bg-stone-100" title="PDF Preview" />
</div>
)}
{hasShoppingList && (() => {
const d = selectedEmail.extracted_data;
const items: any[] = d.items || [];
const vendor: string = d.vendor || '';
const total: number = d.total || 0;
return (
<div className="bg-white p-8 rounded-[3rem] border-2 border-stone-900 shadow-xl flex flex-col">
<div className="flex items-center gap-4 mb-6 text-emerald-600">
<ShoppingCart size={20} />
<span className="font-black text-sm uppercase tracking-widest">
{vendor || 'Receipt Items'}
</span>
</div>
<div className="space-y-2 mb-4">
{items.map((item: any, idx: number) => (
<div key={idx} className="flex justify-between items-center border-b border-stone-50 pb-2 last:border-0">
<span className="text-sm font-bold text-stone-700 truncate mr-3">{item.item_name}</span>
<span className="text-sm font-black text-stone-900 shrink-0">
${(item.price || item.sale_price || 0).toFixed(2)}
</span>
</div>
))}
</div>
{total > 0 && (
<div className="flex justify-between items-center pt-3 border-t border-stone-200 mt-2">
<span className="text-[10px] font-black uppercase tracking-widest text-stone-400">Total</span>
<span className="text-xl font-black text-emerald-600">${total.toFixed(2)}</span>
</div>
)}
</div>
);
})()}
</div>
)}
</div>
</div>
);
})()}
{/* ββ NAV βββββββββββββββββββββββββββββββββββββββββββββββββ */}
<nav className="px-12 py-8 flex items-center justify-between shrink-0 max-w-6xl mx-auto w-full">
<h1
onClick={() => { setView('home'); setSearchResults(null); setSearchQuery(''); }}
className="text-3xl font-serif font-black italic cursor-pointer tracking-tighter hover:opacity-70 transition-opacity"
>
firewall.ai
</h1>
<div className="flex items-center gap-4">
<button onClick={() => setView('sent')} className={`px-6 py-2 rounded-full text-[10px] font-black uppercase tracking-widest transition-colors flex items-center gap-2 ${view === 'sent' ? 'bg-stone-200 text-stone-800' : 'bg-stone-100 text-stone-600 hover:bg-stone-200'}`}>
<History size={14} /> Sent
</button>
<button onClick={() => setIsComposing(true)} className="bg-stone-900 text-white px-6 py-2 rounded-full text-[10px] font-black uppercase tracking-widest hover:bg-emerald-600 transition-colors flex items-center gap-2 shadow-md">
<PenSquare size={14} /> Compose
</button>
<div className="flex items-center gap-3 bg-white pl-2 pr-6 py-2 rounded-full shadow-sm border border-stone-200 ml-4">
<div className="w-8 h-8 rounded-full bg-stone-900 text-emerald-400 flex items-center justify-center font-serif font-black italic text-lg">M</div>
<div className="flex flex-col">
<span className="text-[11px] font-black uppercase tracking-widest text-stone-800 leading-none">Martin</span>
<span className="text-[9px] font-bold text-stone-400 uppercase tracking-wider">Admin</span>
</div>
</div>
</div>
</nav>
{/* ββ MAIN ββββββββββββββββββββββββββββββββββββββββββββββββ */}
<main className="flex-1 flex flex-col w-full max-w-6xl mx-auto px-12">
{/* ββ SENT VIEW βββββββββββββββββββββββββββββββββββββββ */}
{view === 'sent' && (
<div className="w-full flex flex-col animate-in fade-in duration-700 mt-4 pb-12">
<div className="flex items-center gap-3 mb-8">
<History className="text-stone-400" size={28} />
<h2 className="text-4xl font-serif font-black italic text-stone-800 tracking-tighter">Sent History</h2>
</div>
<div className="bg-white rounded-[2rem] border border-stone-200 shadow-sm p-8 flex flex-col min-h-[400px]">
{sentEmails.length === 0 ? (
<p className="text-stone-400 italic">No sent emails.</p>
) : (
<div className="space-y-2">
{sentEmails.map(m => (
<div key={m.id} className="flex items-center gap-4 p-4 hover:bg-stone-50 rounded-[2rem] border border-stone-100 transition-all cursor-pointer" onClick={() => setSelectedEmail(m)}>
<div className="w-10 h-10 shrink-0 rounded-full bg-emerald-50 flex items-center justify-center text-emerald-600 font-serif italic text-lg border border-emerald-100">
<Send size={16} />
</div>
<div className="flex-1 min-w-0 pr-4">
<div className="flex justify-between items-baseline mb-1">
<p className="text-[10px] font-black uppercase text-stone-500 truncate tracking-widest">{m.sender}</p>
<span className="text-[9px] font-bold text-stone-400 shrink-0">{formatShortDate(m.received_at)}</span>
</div>
<p className="text-sm font-bold text-stone-800 truncate">{m.subject}</p>
</div>
</div>
))}
</div>
)}
</div>
</div>
)}
{/* ββ HOME VIEW βββββββββββββββββββββββββββββββββββββββ */}
{view === 'home' && (
<div className="w-full flex flex-col animate-in fade-in duration-700 mt-4">
<form onSubmit={handleSearch} className="relative mb-12 shadow-md rounded-full group">
{isSearching
? <Loader2 className="absolute left-6 top-1/2 -translate-y-1/2 text-emerald-500 animate-spin" size={24} />
: <Search className="absolute left-6 top-1/2 -translate-y-1/2 text-stone-400 group-focus-within:text-emerald-500 transition-colors" size={24} />
}
<input
type="text"
value={searchQuery}
onChange={e => setSearchQuery(e.target.value)}
placeholder="Ask your email anything (e.g. 'What did my landlord say about the lease?')"
className="w-full bg-white border-2 border-stone-200 focus:border-stone-900 rounded-full py-5 pl-16 pr-16 text-stone-800 font-medium outline-none transition-all text-lg shadow-inner"
/>
{searchQuery && (
<button type="button" onClick={() => { setSearchQuery(''); setSearchResults(null); }} className="absolute right-16 top-1/2 -translate-y-1/2 text-stone-400 hover:text-stone-900 p-2">
<X size={18} />
</button>
)}
<button type="submit" className="absolute right-3 top-1/2 -translate-y-1/2 bg-stone-900 text-white p-3 rounded-full hover:bg-emerald-600 transition-colors">
<ArrowRight size={20} />
</button>
</form>
{searchResults ? (
<div className="bg-white rounded-[2rem] border border-stone-200 shadow-sm p-8 flex flex-col min-h-[400px]">
<div className="flex justify-between items-center mb-8 border-b border-stone-100 pb-4">
<h3 className="text-[11px] font-black uppercase tracking-widest text-emerald-600 flex items-center gap-2">
<Search size={14} /> {searchResults.length} Results
</h3>
<button onClick={() => setSearchResults(null)} className="text-[10px] font-black uppercase tracking-widest text-stone-400 hover:text-stone-900 flex items-center gap-1">
<X size={12} /> Clear
</button>
</div>
<div className="grid grid-cols-2 gap-4">
{searchResults.map((m, i) => (
<div key={i} onClick={() => setSelectedEmail(m)} className="relative group flex items-center gap-4 p-4 hover:bg-stone-50 rounded-[2rem] cursor-pointer border border-stone-100 hover:border-stone-300 transition-all">
<div className="w-10 h-10 shrink-0 rounded-full bg-stone-100 flex items-center justify-center text-stone-500 font-serif italic text-lg">
{m.sender.charAt(0).toUpperCase()}
</div>
<div className="flex-1 min-w-0 pr-4">
<p className="text-[10px] font-black uppercase text-stone-400 truncate tracking-widest">{m.sender}</p>
<p className="text-sm font-bold text-stone-800 truncate">{m.subject}</p>
</div>
<div className="absolute right-3 top-1/2 -translate-y-1/2 flex gap-1 opacity-0 group-hover:opacity-100 transition-all bg-white pl-2">
<div className="relative">
<button onClick={ev => { ev.stopPropagation(); setRetagMenuId(retagMenuId === m.id ? null : m.id); }} className="p-1.5 text-stone-300 hover:text-emerald-500 hover:bg-emerald-50 rounded-full border border-transparent hover:border-stone-100 shadow-sm" title="Move to...">
<MoveRight size={14} />
</button>
{retagMenuId === m.id && (
<div onClick={ev => ev.stopPropagation()} className="absolute right-0 top-8 bg-white border-2 border-stone-900 rounded-2xl shadow-xl z-50 overflow-hidden min-w-[160px]">
<p className="text-[9px] font-black uppercase tracking-widest text-stone-400 px-4 pt-3 pb-1 text-left">File as Intelligence</p>
{/* π’ MAP OVER MOVE_MENU_TABS INSTEAD OF VAULT_TABS */}
{MOVE_MENU_TABS.filter(t => t.id !== 'ARCHIVED' && t.id !== 'SENT' && t.id !== m.librarian_tag).map(t => {
const TIcon = t.icon;
return (
<button key={t.id} onClick={() => handleRetag(m.id, t.id, m.librarian_tag)} className="w-full flex items-center gap-2 px-4 py-2.5 text-xs font-bold text-stone-700 hover:bg-stone-50 transition-colors border-b last:border-0 border-stone-50">
<TIcon size={12} className="text-stone-400" /> {t.label}
</button>
);
})}
</div>
)}
</div>
<button onClick={ev => { ev.stopPropagation(); handleDelete(m.id); }} className="p-1.5 text-stone-300 hover:text-red-500 hover:bg-red-100 rounded-full"><Trash2 size={14} /></button>
</div>
</div>
))}
</div>
</div>
) : (
<>
<div className="grid grid-cols-3 gap-8 mb-8">
{/* Direct Messages */}
<div className="col-span-2 bg-white rounded-[2rem] border border-stone-200 shadow-sm p-6 flex flex-col h-[340px]">
<h3 className="text-[10px] font-black uppercase tracking-widest text-stone-400 flex items-center gap-2 mb-4 ml-2">
<User size={14} /> Direct Messages
</h3>
<div className="space-y-1 overflow-y-auto pr-2">
{feed.filter(e => e.is_human && !archivedIds.has(e.id)).slice(0, 8).map((m, i) => (
<div key={i} onClick={() => setSelectedEmail(m)} className="relative group flex items-center gap-4 p-3 hover:bg-stone-50 rounded-2xl cursor-pointer transition-colors border border-transparent hover:border-stone-100">
<div className="w-10 h-10 shrink-0 rounded-full bg-stone-100 flex items-center justify-center text-stone-500 font-serif italic text-lg">
{m.sender.charAt(0).toUpperCase()}
</div>
<div className="flex-1 min-w-0 pr-4">
<div className="flex justify-between items-baseline">
<p className="text-[10px] font-black uppercase text-emerald-600 truncate tracking-widest">{m.sender.split('@')[0]}</p>
<span className="text-[9px] font-bold text-stone-400 shrink-0 ml-2">{formatShortDate(m.received_at)}</span>
</div>
<p className="text-sm font-bold text-stone-800 truncate">{m.subject}</p>
</div>
</div>
))}
</div>
</div>
{/* Auth Events */}
<div className="col-span-1 bg-white rounded-[2rem] border border-stone-200 shadow-sm p-6 flex flex-col h-[340px]">
<h3 className="text-[10px] font-black uppercase tracking-widest text-stone-400 flex items-center gap-2 mb-4 ml-2">
<ShieldAlert size={14} /> Auth Events
</h3>
<div className="space-y-1 overflow-y-auto pr-2">
{feed.filter(e => e.is_key && !archivedIds.has(e.id)).slice(0, 8).map((m, i) => (
<div key={i} onClick={() => setSelectedEmail(m)} className="relative group flex items-center gap-4 p-3 hover:bg-stone-50 rounded-2xl cursor-pointer border border-transparent hover:border-stone-100 transition-colors">
<div className="w-10 h-10 shrink-0 rounded-full bg-stone-50 flex items-center justify-center border border-stone-100">
{getAuthIcon(m.sender, m.subject)}
</div>
<div className="flex-1 min-w-0 pr-4">
<div className="flex justify-between items-baseline">
<p className="text-[10px] font-black uppercase text-stone-400 truncate tracking-widest">{m.sender.split('@')[0]}</p>
<span className="text-[9px] font-bold text-stone-300 shrink-0 ml-2">{formatShortDate(m.received_at)}</span>
</div>
<p className="text-sm font-bold text-stone-800 truncate">{m.subject}</p>
</div>
</div>
))}
</div>
</div>
</div>
{/* Vault CTA */}
<div onClick={() => setView('vault')} className="w-full bg-stone-900 rounded-[2.5rem] shadow-xl p-8 flex flex-col items-center justify-center cursor-pointer hover:bg-emerald-950 transition-colors group relative overflow-hidden border border-stone-800">
<div className="absolute inset-0 bg-[radial-gradient(ellipse_at_top,_var(--tw-gradient-stops))] from-stone-800 via-stone-900 to-stone-900 opacity-50" />
<div className="flex flex-col items-center relative z-10 w-full">
<div className="flex items-center gap-4 mb-3">
<TrendingUp size={36} className="text-emerald-400 group-hover:scale-110 transition-transform" />
<h2 className="text-4xl font-serif font-black italic text-white">The Vault</h2>
</div>
<p className="text-[10px] font-black uppercase text-emerald-500 tracking-[0.4em] mb-6 italic">Intelligence Analysis Engine</p>
<div className="flex gap-3 flex-wrap justify-center max-w-4xl">
{["Finance", "Receipts", "Deals", "Logistics", "Career", "Alerts", "Newsletters"].map(cat => (
<span key={cat} className="text-[10px] font-black text-stone-300 uppercase tracking-widest border border-stone-700 bg-stone-800/50 px-4 py-1.5 rounded-full shadow-inner flex items-center gap-1.5">
{cat}
</span>
))}
</div>
</div>
</div>
</>
)}
</div>
)}
{/* ββ VAULT VIEW βββββββββββββββββββββββββββββββββββββββ */}
{view === 'vault' && (
<div className="w-full flex-1 flex flex-col animate-in slide-in-from-bottom-8 duration-500 pb-12 mt-4">
{/* Tab bar */}
<div className="flex gap-2 mb-8 bg-white border border-stone-200 p-2 rounded-full self-center shadow-sm overflow-x-auto max-w-full">
{VAULT_TABS.map(tab => {
const Icon = tab.icon;
const isActive = activeTab === tab.id;
const count = tab.id === 'ARCHIVED'
? archivedIds.size
: feed.filter(e => e.librarian_tag === tab.id && !archivedIds.has(e.id)).length;
return (
<button
key={tab.id}
onClick={() => setActiveTab(tab.id)}
onDragOver={e => {
e.preventDefault();
if (draggedEmail) setDragOverTab(tab.id);
}}
onDragLeave={() => setDragOverTab(null)}
onDrop={e => {
e.preventDefault();
setDragOverTab(null);
if (draggedEmail && tab.id !== draggedEmail.librarian_tag) {
handleRetag(draggedEmail.id, tab.id, draggedEmail.librarian_tag);
}
setDraggedEmail(null);
}}
className={`px-6 py-2.5 rounded-full text-[10px] font-black uppercase tracking-widest transition-all whitespace-nowrap flex items-center gap-1.5
${isActive ? 'bg-stone-900 text-white shadow-md' : 'text-stone-400 hover:text-stone-950'}
${dragOverTab === tab.id ? 'ring-2 ring-emerald-400 bg-emerald-50 text-emerald-700 scale-105' : ''}
`}
>
<Icon size={11} /> {tab.label}
{count > 0 && (
<span className={`text-[8px] font-black px-1.5 py-0.5 rounded-full ml-0.5 ${isActive ? 'bg-white/20 text-white' : 'bg-stone-100 text-stone-500'}`}>
{count}
</span>
)}
</button>
);
})}
</div>
{/* Vault body */}
<div className="bg-white border-2 border-stone-200 rounded-[4rem] flex-1 flex overflow-hidden shadow-xl h-[70vh]">
{/* Intelligence left panel */}
{activeTab !== 'ARCHIVED' && (
<div className="w-[480px] p-12 border-r border-stone-100 overflow-y-auto bg-stone-50/50">
<IntelPanel activeTab={activeTab} intel={intel} feed={feed} setSelectedEmail={setSelectedEmail} archivedIds={archivedIds} />
</div>
)}
{/* Email feed right panel */}
<div className="flex-1 p-12 overflow-y-auto bg-white">
{Object.entries(groupedFeed).map(([groupName, items]) =>
items.length > 0 ? (
<div key={groupName} className="mb-10">
<h3 className="text-[10px] font-black uppercase tracking-[0.3em] text-stone-400 mb-5 border-b border-stone-100 pb-3">{groupName}</h3>
<div className="grid grid-cols-2 gap-5">
{items.map((e, i) => {
const logoUrl = getLogo(e.sender);
const initial = e.sender.charAt(0).toUpperCase();
return (
<div
key={i}
onClick={() => { if (retagMenuId === e.id) { setRetagMenuId(null); return; } setSelectedEmail(e); }}
draggable
onDragStart={() => { setDraggedEmail(e); setRetagMenuId(null); }}
onDragEnd={() => setDraggedEmail(null)}
className={`relative group bg-white p-8 rounded-[3rem] border transition-all flex flex-col items-center text-center cursor-pointer
${isRetagging === e.id ? 'border-emerald-300 bg-emerald-50/30 animate-pulse' : 'border-stone-100 hover:border-stone-900 hover:shadow-lg'}
${draggedEmail?.id === e.id ? 'opacity-40 scale-95' : ''}
`}
>
{/* Top-right action bar */}
<div className="absolute right-4 top-4 flex gap-1 opacity-0 group-hover:opacity-100 transition-all z-10">
<div className="p-2 text-stone-300 cursor-grab active:cursor-grabbing">
<GripVertical size={14} />
</div>
<div className="relative">
<button
onClick={ev => { ev.stopPropagation(); setRetagMenuId(retagMenuId === e.id ? null : e.id); }}
className="p-2 text-stone-300 hover:text-emerald-500 hover:bg-emerald-50 rounded-full bg-white shadow-sm"
title="Move to..."
>
<MoveRight size={14} />
</button>
{retagMenuId === e.id && (
<div
onClick={ev => ev.stopPropagation()}
className="absolute right-0 top-8 bg-white border-2 border-stone-900 rounded-2xl shadow-xl z-50 overflow-hidden min-w-[160px]"
>
<p className="text-[9px] font-black uppercase tracking-widest text-stone-400 px-4 pt-3 pb-1 text-left">File as Intelligence</p>
{/* π’ MAP OVER MOVE_MENU_TABS INSTEAD OF VAULT_TABS */}
{MOVE_MENU_TABS.filter(t => t.id !== e.librarian_tag).map(t => {
const TIcon = t.icon;
return (
<button
key={t.id}
onClick={() => handleRetag(e.id, t.id, e.librarian_tag)}
className="w-full flex items-center gap-2 px-4 py-2.5 text-xs font-bold text-stone-700 hover:bg-stone-50 transition-colors border-b last:border-0 border-stone-50"
>
<TIcon size={12} className={t.id === 'ARCHIVED' ? "text-stone-400" : "text-emerald-500"} /> {t.label}
</button>
);
})}
</div>
)}
</div>
<button onClick={ev => { ev.stopPropagation(); toggleArchive(e.id); }} className="p-2 text-stone-300 hover:text-emerald-500 hover:bg-emerald-50 rounded-full bg-white shadow-sm">
<Archive size={14} />
</button>
<button onClick={ev => { ev.stopPropagation(); handleDelete(e.id); }} className="p-2 text-stone-300 hover:text-red-500 hover:bg-red-50 rounded-full bg-white shadow-sm">
<Trash2 size={14} />
</button>
</div>
<div className="h-10 mb-4 flex items-center justify-center w-full">
{logoUrl ? (
<img
src={logoUrl}
className="h-full object-contain grayscale group-hover:grayscale-0 transition-all duration-300"
onError={el => el.currentTarget.style.display = 'none'}
alt=""
/>
) : (
<div className="w-12 h-12 bg-stone-900 rounded-full flex items-center justify-center text-white text-xl font-serif italic shadow-sm">{initial}</div>
)}
</div>
<span className="text-[9px] font-black text-stone-300 uppercase tracking-widest mb-1">{formatShortDate(e.received_at)}</span>
<span className="text-[9px] font-black text-stone-400 uppercase tracking-widest mb-2 truncate w-full group-hover:text-emerald-500 transition-colors">{e.sender}</span>
<p className="text-sm font-bold text-stone-800 leading-tight tracking-tight line-clamp-2">{e.subject}</p>
</div>
);
})}
</div>
</div>
) : null
)}
{displayFeed.length === 0 && (
<div className="h-full flex flex-col items-center justify-center text-stone-300 opacity-50">
<Archive size={48} />
<p className="font-black italic mt-4 tracking-widest uppercase text-sm">No items found</p>
</div>
)}
</div>
</div>
</div>
)}
</main>
</div>
);
}
import os
import sys
import psycopg2
from psycopg2.extras import RealDictCursor
from fastapi import FastAPI, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from typing import Optional
from dotenv import load_dotenv
from sentence_transformers import SentenceTransformer
# Import the worker function directly for native background execution
from intelligence_worker import run_domain
load_dotenv()
DB_HOST = os.getenv("DB_HOST", "localhost")
DB_NAME = os.getenv("DB_NAME", "postgres")
DB_USER = os.getenv("DB_USER", "postgres")
DB_PASS = os.getenv("DB_PASS")
app = FastAPI(title="firewall.ai Executive API")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
ATTACHMENT_DIR = os.path.join(os.getcwd(), "saved_attachments")
os.makedirs(ATTACHMENT_DIR, exist_ok=True)
app.mount("/attachments", StaticFiles(directory=ATTACHMENT_DIR), name="attachments")
print("[INIT] Loading embedding model for search (all-mpnet-base-v2)...")
embed_model = SentenceTransformer('all-mpnet-base-v2')
def get_db_connection():
return psycopg2.connect(
host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS,
cursor_factory=RealDictCursor
)
class EmailDraft(BaseModel):
to_address: str
cc_address: Optional[str] = ""
bcc_address: Optional[str] = ""
subject: str
body: str
reply_to_id: Optional[int] = None
is_forward: Optional[bool] = False
@app.get("/api/intelligence")
def get_daily_intelligence():
"""Fetches the latest structured JSON from all 6 domain tables."""
conn = get_db_connection()
cur = conn.cursor()
try:
intel_payload = {}
tables = [
("intel_finance", "FINANCE"),
("intel_groceries", "GROCERIES"),
("intel_logistics", "LOGISTICS"),
("intel_career", "CAREER"),
("intel_notifications", "NOTIFICATION"),
("intel_newsletters", "NEWSLETTER")
]
for table, key in tables:
cur.execute(f"SELECT * FROM {table} ORDER BY report_date DESC LIMIT 1;")
row = cur.fetchone()
if row:
# Strip metadata columns - only pass pure insight JSON to frontend
row.pop("report_date", None)
row.pop("last_updated", None)
intel_payload[key] = row
return intel_payload
except Exception as e:
print(f"[ERROR] Fetching intelligence: {e}")
return {}
finally:
cur.close()
conn.close()
@app.get("/api/live-feed")
def get_live_feed():
conn = get_db_connection()
cur = conn.cursor()
try:
cur.execute("""
SELECT e.id, e.subject, e.sender, e.body_text, e.extracted_data,
e.is_human, e.is_key, e.librarian_tag, e.received_at,
a.file_hash, a.file_name
FROM emails e
LEFT JOIN attachments a ON e.id = a.email_id
ORDER BY e.received_at DESC LIMIT 100;
""")
return cur.fetchall()
finally:
cur.close()
conn.close()
@app.get("/api/search")
def search_emails(q: str):
if not q.strip(): return []
conn = get_db_connection()
cur = conn.cursor()
try:
query_vector = embed_model.encode(q).tolist()
cur.execute("""
SELECT e.id, e.subject, e.sender, e.body_text, e.extracted_data,
e.is_human, e.is_key, e.librarian_tag, e.received_at,
a.file_hash, a.file_name,
1 - (em.embedding <=> %s::vector) AS similarity_score
FROM emails e
JOIN embeddings em ON e.id = em.email_id
LEFT JOIN attachments a ON e.id = a.email_id
ORDER BY em.embedding <=> %s::vector
LIMIT 15;
""", (query_vector, query_vector))
return cur.fetchall()
except Exception as e:
print(f"[ERROR] Vector search failed: {e}")
return []
finally:
cur.close()
conn.close()
@app.post("/api/send")
def send_email(draft: EmailDraft):
print(f"\n[DISPATCH] Secure outgoing message to {draft.to_address}")
return {"status": "success", "message": "Message securely dispatched."}
@app.delete("/api/email/{email_id}")
def delete_email(email_id: int):
conn = get_db_connection()
cur = conn.cursor()
try:
cur.execute("DELETE FROM attachments WHERE email_id = %s;", (email_id,))
cur.execute("DELETE FROM embeddings WHERE email_id = %s;", (email_id,))
cur.execute("DELETE FROM emails WHERE id = %s;", (email_id,))
conn.commit()
return {"status": "success"}
except Exception as e:
conn.rollback()
return {"status": "error"}
finally:
cur.close()
conn.close()
class RetagRequest(BaseModel):
new_tag: str
old_tag: str
@app.patch("/api/email/{email_id}/retag")
def retag_email(email_id: int, req: RetagRequest, background_tasks: BackgroundTasks):
"""
Re-tags an email and triggers targeted intelligence re-processing
via native FastAPI background tasks to ensure rapid UI updates.
"""
valid_tags = ["FINANCE", "RECEIPT", "PROMOTION", "LOGISTICS", "CAREER", "NOTIFICATION", "NEWSLETTER", "ARCHIVED", "HUMAN", "KEY"]
new_tag = req.new_tag.upper()
old_tag = req.old_tag.upper()
if new_tag not in valid_tags:
return {"status": "error", "message": f"Invalid tag: {new_tag}"}
# Force the booleans to update so it completely leaves the Home screen if dragged to Vault
is_human = True if new_tag == "HUMAN" else False
is_key = True if new_tag == "KEY" else False
conn = get_db_connection()
cur = conn.cursor()
try:
# Update BOTH the tag and the booleans
cur.execute("""
UPDATE emails
SET librarian_tag = %s, is_human = %s, is_key = %s
WHERE id = %s
""", (new_tag, is_human, is_key, email_id))
conn.commit()
print(f"\n[RETAG] Email {email_id}: {old_tag} -> {new_tag}")
except Exception as e:
conn.rollback()
return {"status": "error", "message": str(e)}
finally:
cur.close()
conn.close()
# Re-run intelligence for both affected domains natively in the background
worker_domains = {"FINANCE", "RECEIPT", "PROMOTION", "LOGISTICS", "CAREER", "NOTIFICATION", "NEWSLETTER"}
tags_to_reprocess = [t for t in {old_tag, new_tag} if t in worker_domains]
for tag in tags_to_reprocess:
print(f" [QUEUE] Background recalculation triggered for domain: {tag}")
background_tasks.add_task(run_domain, tag)
return {
"status": "success",
"email_id": email_id,
"new_tag": new_tag,
"old_tag": old_tag,
"reprocessing": tags_to_reprocess,
}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)
Tech Stack
- > AI Models: Qwen3:8b (Local / Ollama), all-mpnet-base-v2 (Embeddings)
- > Database: PostgreSQL + pgvector
- > Backend: FastAPI, Psycopg2, pypdf
- > Frontend: React, TailwindCSS, Lucide Icons