From 7a8790d564227d596138af30e22d710438cbe4ff Mon Sep 17 00:00:00 2001 From: Francisco Pessano Date: Sat, 7 Jun 2025 00:59:09 -0300 Subject: [PATCH] Add AI parsing capabilities for debt information and update database schema - Implemented AI-powered parsing for debt emails using Google's Gemini model. - Enhanced Postmark webhook to extract debt details including amount, vendor, description, due date, and legitimacy. - Updated database schema to include new columns for AI-extracted data. - Added environment variable requirements and updated package dependencies. - Created migration script for new database columns and indexes. --- .env.example | 10 + .gitignore | 4 + AI_PARSING_UPDATE.md | 98 +++++++ package-lock.json | 243 ++++++++++++++++- package.json | 5 +- src/pages/api/postmark.ts | 250 ++++++++++++------ .../20250607000500_add_ai_parsing_columns.sql | 16 ++ 7 files changed, 529 insertions(+), 97 deletions(-) create mode 100644 .env.example create mode 100644 AI_PARSING_UPDATE.md create mode 100644 supabase/migrations/20250607000500_add_ai_parsing_columns.sql diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c10b0cf --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +# Environment variables for Inbox Negotiator + +# Supabase Configuration +SUPABASE_URL=your_supabase_url_here +SUPABASE_ANON_KEY=your_supabase_anon_key_here + +# Google Generative AI API Key for Gemini model +GOOGLE_GENERATIVE_AI_API_KEY=your_google_api_key_here + +# Add these same variables to your actual .env file diff --git a/.gitignore b/.gitignore index 016b59e..a40b315 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ pnpm-debug.log* # jetbrains setting folder .idea/ + +# supabase files +supabase/.branches +supabase/.temp \ No newline at end of file diff --git a/AI_PARSING_UPDATE.md b/AI_PARSING_UPDATE.md new file mode 100644 index 0000000..6e1d0bf --- /dev/null +++ b/AI_PARSING_UPDATE.md @@ -0,0 +1,98 @@ +# AI-Enhanced Debt Parsing Update + +## Changes Made + +### 1. Postmark Endpoint Enhancement (`src/pages/api/postmark.ts`) + +The Postmark webhook endpoint has been enhanced to use Vercel's AI package with Google's Gemini model for intelligent debt parsing. + +#### Key Improvements: +- **AI-Powered Parsing**: Replaced regex-based amount extraction with Gemini 1.5 Flash model +- **Enhanced Data Extraction**: Now extracts: + - Debt amount (more accurate than regex) + - Vendor/creditor name + - Description of what the debt is for + - Due date (if mentioned) + - Whether it's a legitimate debt collection notice + +#### Fallback Mechanism: +- If AI parsing fails or API key is not configured, falls back to original regex parsing +- Ensures system reliability even if AI service is unavailable + +### 2. Database Schema Updates + +Added new columns to the `debts` table: +- `description` (text) - AI-extracted description +- `due_date` (timestamptz) - Extracted due date +- `metadata` (jsonb) - Additional AI-extracted information + +### 3. Dependencies Added + +```bash +npm install ai @ai-sdk/google +``` + +### 4. Environment Variables Required + +Add to your `.env` file: +```bash +GOOGLE_GENERATIVE_AI_API_KEY=your_google_api_key_here +SUPABASE_URL=your_supabase_url +SUPABASE_ANON_KEY=your_supabase_anon_key +``` + +## How to Get Google API Key + +1. Go to [Google AI Studio](https://aistudio.google.com/) +2. Create a new API key +3. Add it to your environment variables as `GOOGLE_GENERATIVE_AI_API_KEY` + +## Testing the Changes + +### 1. Test Email Processing +Send a test email to your Postmark webhook endpoint with debt-related content like: +``` +Subject: Outstanding Balance Notice + +Dear Customer, + +You have an outstanding balance of $150.00 for your account with XYZ Collections. +This amount is due by January 15, 2025. + +Please contact us to arrange payment. +``` + +### 2. Expected Database Entry +The system should now create a debt record with: +- `amount`: 150.00 +- `vendor`: sender's email +- `description`: AI-generated description +- `due_date`: 2025-01-15 (if extracted) +- `metadata`: JSON with isDebtCollection flag and other details + +### 3. Migration Application +If using local Supabase, apply the migration: +```bash +supabase db reset +# or +supabase migration up +``` + +For production, apply the migration found in: +`supabase/migrations/20250607000500_add_ai_parsing_columns.sql` + +## Benefits + +1. **More Accurate Parsing**: AI can understand context better than regex +2. **Richer Data**: Extracts more information from emails +3. **Better Classification**: Determines if email is actually a debt collection notice +4. **Future-Proof**: Can be enhanced with more sophisticated AI models +5. **Reliable Fallback**: Still works if AI service is unavailable + +## Next Steps + +1. Set up Google API key +2. Apply database migration +3. Test with sample debt collection emails +4. Monitor logs for AI parsing accuracy +5. Consider training on domain-specific examples for better accuracy diff --git a/package-lock.json b/package-lock.json index a609b5b..cba19a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,8 +8,10 @@ "name": "@example/basics", "version": "0.0.1", "dependencies": { + "@ai-sdk/google": "^1.2.19", "@astrojs/react": "^3.6.2", "@astrojs/tailwind": "^5.1.1", + "@google/generative-ai": "^0.24.1", "@hookform/resolvers": "^3.9.0", "@radix-ui/react-accordion": "^1.2.0", "@radix-ui/react-alert-dialog": "^1.1.1", @@ -41,6 +43,7 @@ "@supabase/supabase-js": "^2.39.0", "@types/react": "^18.3.10", "@types/react-dom": "^18.3.0", + "ai": "^4.3.16", "astro": "^4.15.9", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", @@ -64,6 +67,92 @@ "zod": "^3.23.8" } }, + "node_modules/@ai-sdk/google": { + "version": "1.2.19", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-1.2.19.tgz", + "integrity": "sha512-Xgl6eftIRQ4srUdCzxM112JuewVMij5q4JLcNmHcB68Bxn9dpr3MVUSPlJwmameuiQuISIA8lMB+iRiRbFsaqA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", + "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", + "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz", + "integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/ui-utils": "1.2.11", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/ui-utils": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", + "integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", @@ -886,6 +975,15 @@ "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.8.tgz", "integrity": "sha512-kym7SodPp8/wloecOpcmSnWJsK7M0E5Wg8UcFA+uO4B9s5d0ywXOEro/8HM9x0rW+TljRzul/14UYz3TleT3ig==" }, + "node_modules/@google/generative-ai": { + "version": "0.24.1", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", + "integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@hookform/resolvers": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/@hookform/resolvers/-/resolvers-3.9.0.tgz", @@ -1375,6 +1473,15 @@ "node": ">= 8" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@oslojs/encoding": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@oslojs/encoding/-/encoding-1.1.0.tgz", @@ -3164,6 +3271,12 @@ "@types/ms": "*" } }, + "node_modules/@types/diff-match-patch": { + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", + "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -3283,6 +3396,32 @@ "node": ">=0.4.0" } }, + "node_modules/ai": { + "version": "4.3.16", + "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.16.tgz", + "integrity": "sha512-KUDwlThJ5tr2Vw0A1ZkbDKNME3wzWhuVfAOwIvFUzl1TPVDFAXDFTXio3p+jaKneB+dKNCvFFlolYmmgHttG1g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/react": "1.2.12", + "@ai-sdk/ui-utils": "1.2.11", + "@opentelemetry/api": "1.9.0", + "jsondiffpatch": "0.6.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, "node_modules/ansi-align": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", @@ -4534,6 +4673,12 @@ "node": ">=0.3.1" } }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", + "license": "Apache-2.0" + }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", @@ -5390,6 +5535,12 @@ "node": ">=6" } }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -5401,6 +5552,35 @@ "node": ">=6" } }, + "node_modules/jsondiffpatch": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.6.0.tgz", + "integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==", + "license": "MIT", + "dependencies": { + "@types/diff-match-patch": "^1.0.36", + "chalk": "^5.3.0", + "diff-match-patch": "^1.0.5" + }, + "bin": { + "jsondiffpatch": "bin/jsondiffpatch.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/jsondiffpatch/node_modules/chalk": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", + "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -6425,15 +6605,16 @@ } }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -7509,6 +7690,12 @@ "node": ">=4" } }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -7828,6 +8015,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/swr": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.3.3.tgz", + "integrity": "sha512-dshNvs3ExOqtZ6kJBaAsabhPdHyeY4P2cKwRCniDVifBMoG/SVI7tfLWqPXriVspf2Rg4tPzXJTnwaihIeFw2A==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.4.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/tailwind-merge": { "version": "2.5.3", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.5.3.tgz", @@ -7919,6 +8119,18 @@ "node": ">=0.8" } }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/tiny-invariant": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", @@ -8238,6 +8450,15 @@ } } }, + "node_modules/use-sync-external-store": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.5.0.tgz", + "integrity": "sha512-Rb46I4cGGVBmjamjphe8L/UnvJD+uPPtTkNvX5mZgqdbavhI4EbgIWJiIHXJ8bc/i9EQGPRh4DwEURJ552Do0A==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -8635,19 +8856,21 @@ } }, "node_modules/zod": { - "version": "3.23.8", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", - "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "version": "3.25.56", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.56.tgz", + "integrity": "sha512-rd6eEF3BTNvQnR2e2wwolfTmUTnp70aUTqr0oaGbHifzC3BKJsoV+Gat8vxUMR1hwOKBs6El+qWehrHbCpW6SQ==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, "node_modules/zod-to-json-schema": { - "version": "3.23.3", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.3.tgz", - "integrity": "sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog==", + "version": "3.24.5", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz", + "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", + "license": "ISC", "peerDependencies": { - "zod": "^3.23.3" + "zod": "^3.24.1" } }, "node_modules/zod-to-ts": { diff --git a/package.json b/package.json index 86a717b..0a1750d 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,10 @@ "astro": "astro" }, "dependencies": { + "@ai-sdk/google": "^1.2.19", "@astrojs/react": "^3.6.2", "@astrojs/tailwind": "^5.1.1", + "@google/generative-ai": "^0.24.1", "@hookform/resolvers": "^3.9.0", "@radix-ui/react-accordion": "^1.2.0", "@radix-ui/react-alert-dialog": "^1.1.1", @@ -44,6 +46,7 @@ "@supabase/supabase-js": "^2.39.0", "@types/react": "^18.3.10", "@types/react-dom": "^18.3.0", + "ai": "^4.3.16", "astro": "^4.15.9", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", @@ -66,4 +69,4 @@ "vaul": "^1.0.0", "zod": "^3.23.8" } -} \ No newline at end of file +} diff --git a/src/pages/api/postmark.ts b/src/pages/api/postmark.ts index 0190293..5a9d34c 100644 --- a/src/pages/api/postmark.ts +++ b/src/pages/api/postmark.ts @@ -1,98 +1,176 @@ -import type { APIRoute } from 'astro'; -import { supabase } from '../../lib/supabase'; +import type { APIRoute } from "astro"; +import { supabase } from "../../lib/supabase"; +import { generateObject } from "ai"; +import { + createGoogleGenerativeAI, + google, + type GoogleGenerativeAIProviderOptions, +} from "@ai-sdk/google"; +import { z } from "zod"; + +// Schema for debt information extraction +const debtSchema = z.object({ + amount: z.number().min(0).describe("The debt amount in dollars"), + vendor: z.string().describe("The name or identifier of the vendor/creditor"), + description: z.string().describe("Brief description of what the debt is for"), + dueDate: z.string().optional().describe("Due date if mentioned (ISO format)"), + isDebtCollection: z + .boolean() + .describe("Whether this appears to be a debt collection notice"), +}); + +// Function to parse debt information using AI +async function parseDebtWithAI(emailText: string, fromEmail: string) { + try { + // Check if Google API key is available + const googleApiKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY; + if (!googleApiKey) { + console.warn( + "Google API key not configured, falling back to regex parsing" + ); + throw new Error("No Google API key configured"); + } + + const result = await generateObject({ + model: createGoogleGenerativeAI({ + apiKey: googleApiKey, + })("gemini-2.5-flash-preview-04-17"), + system: `You are an expert at analyzing debt collection and billing emails. + Extract key debt information from the email content. + Look for monetary amounts, creditor information, what the debt is for, and due dates. + If this doesn't appear to be a legitimate debt or billing notice, set amount to 0. + Be very accurate with amounts - look for dollar signs and numbers carefully.`, + prompt: `Parse this email for debt information: + + From: ${fromEmail} + Content: ${emailText}`, + schema: debtSchema, + }); + + return result.object; + } catch (error) { + console.error("AI parsing error:", error); + // Fallback to regex if AI fails + const amountMatch = emailText.match(/\$(\d+(?:,\d{3})*(?:\.\d{2})?)/); + return { + amount: amountMatch ? parseFloat(amountMatch[1].replace(/,/g, "")) : 0, + vendor: fromEmail || "unknown", + description: "Failed to parse with AI - using regex fallback", + isDebtCollection: amountMatch ? true : false, + }; + } +} export const POST: APIRoute = async ({ request }) => { - try { - const data = await request.json(); - - // Check for opt-out keywords - const optOutKeywords = ['STOP', 'UNSUBSCRIBE', 'OPT-OUT', 'REMOVE']; - const textBody = data.TextBody || ''; - const hasOptOut = optOutKeywords.some(keyword => - textBody.toUpperCase().includes(keyword) - ); + try { + const data = await request.json(); - if (hasOptOut) { - // Log opt-out and don't process further - const { error } = await supabase.from('debts').insert({ - vendor: data.FromFull?.Email || 'unknown', - amount: 0, - raw_email: textBody, - status: 'opted_out' - }); + // Check for opt-out keywords + const optOutKeywords = ["STOP", "UNSUBSCRIBE", "OPT-OUT", "REMOVE"]; + const textBody = data.TextBody || ""; + const fromEmail = data.FromFull?.Email || "unknown"; - if (error) { - console.error('Error logging opt-out:', error); - return new Response(JSON.stringify({ error: error.message }), { - status: 500, - headers: { 'Content-Type': 'application/json' } - }); - } + const hasOptOut = optOutKeywords.some((keyword) => + textBody.toUpperCase().includes(keyword) + ); - return new Response('Opt-out processed', { status: 200 }); - } + if (hasOptOut) { + // Log opt-out and don't process further + const { error } = await supabase.from("debts").insert({ + vendor: fromEmail, + amount: 0, + raw_email: textBody, + status: "opted_out", + }); - // Extract debt amount using regex - const amountMatch = textBody.match(/\$(\d+(?:,\d{3})*(?:\.\d{2})?)/); - const amount = amountMatch ? parseFloat(amountMatch[1].replace(/,/g, '')) : 0; + if (error) { + console.error("Error logging opt-out:", error); + return new Response(JSON.stringify({ error: error.message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } - // Insert debt record - const { data: insertedDebt, error: insertError } = await supabase - .from('debts') - .insert({ - vendor: data.FromFull?.Email || 'unknown', - amount: amount, - raw_email: textBody, - status: 'received' - }) - .select() - .single(); + return new Response("Opt-out processed", { status: 200 }); + } - if (insertError) { - console.error('Error inserting debt:', insertError); - return new Response(JSON.stringify({ error: insertError.message }), { - status: 500, - headers: { 'Content-Type': 'application/json' } - }); - } + // Parse debt information using AI + const debtInfo = await parseDebtWithAI(textBody, fromEmail); - // Log the email receipt - await supabase.from('audit_logs').insert({ - debt_id: insertedDebt.id, - action: 'email_received', - details: { - vendor: data.FromFull?.Email, - amount: amount, - subject: data.Subject - } - }); + // Insert debt record with AI-extracted information + const { data: insertedDebt, error: insertError } = await supabase + .from("debts") + .insert({ + vendor: debtInfo.vendor, + amount: debtInfo.amount, + raw_email: textBody, + status: "received", + description: debtInfo.description, + due_date: debtInfo.dueDate, + metadata: { + isDebtCollection: debtInfo.isDebtCollection, + subject: data.Subject, + fromEmail: fromEmail, + }, + }) + .select() + .single(); - // Trigger negotiation function - if (amount > 0) { - const negotiateUrl = `${import.meta.env.SUPABASE_URL}/functions/v1/negotiate`; - - try { - await fetch(negotiateUrl, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${import.meta.env.SUPABASE_ANON_KEY}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ record: insertedDebt }) - }); - } catch (negotiateError) { - console.error('Error triggering negotiation:', negotiateError); - // Don't fail the webhook if negotiation fails - } - } + if (insertError) { + console.error("Error inserting debt:", insertError); + return new Response(JSON.stringify({ error: insertError.message }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } - return new Response('OK', { status: 200 }); + // Log the email receipt + await supabase.from("audit_logs").insert({ + debt_id: insertedDebt.id, + action: "email_received", + details: { + vendor: debtInfo.vendor, + amount: debtInfo.amount, + subject: data.Subject, + aiParsed: true, + }, + }); - } catch (error) { - console.error('Postmark webhook error:', error); - return new Response(JSON.stringify({ error: 'Internal server error' }), { - status: 500, - headers: { 'Content-Type': 'application/json' } - }); - } -}; \ No newline at end of file + // Trigger negotiation function if this is a legitimate debt + if (debtInfo.amount > 0 && debtInfo.isDebtCollection) { + // Access environment variables through Astro runtime + const supabaseUrl = process.env.SUPABASE_URL; + const supabaseAnonKey = process.env.SUPABASE_ANON_KEY; + + if (supabaseUrl && supabaseAnonKey) { + const negotiateUrl = `${supabaseUrl}/functions/v1/negotiate`; + + try { + await fetch(negotiateUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${supabaseAnonKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ record: insertedDebt }), + }); + } catch (negotiateError) { + console.error("Error triggering negotiation:", negotiateError); + // Don't fail the webhook if negotiation fails + } + } else { + console.warn( + "Supabase environment variables not configured for negotiation trigger" + ); + } + } + + return new Response("OK", { status: 200 }); + } catch (error) { + console.error("Postmark webhook error:", error); + return new Response(JSON.stringify({ error: "Internal server error" }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +}; diff --git a/supabase/migrations/20250607000500_add_ai_parsing_columns.sql b/supabase/migrations/20250607000500_add_ai_parsing_columns.sql new file mode 100644 index 0000000..663990d --- /dev/null +++ b/supabase/migrations/20250607000500_add_ai_parsing_columns.sql @@ -0,0 +1,16 @@ +-- Add new columns for AI-enhanced debt parsing +-- Migration for improved debt information storage + +-- Add new columns to debts table +ALTER TABLE debts ADD COLUMN IF NOT EXISTS description text; +ALTER TABLE debts ADD COLUMN IF NOT EXISTS due_date timestamptz; +ALTER TABLE debts ADD COLUMN IF NOT EXISTS metadata jsonb DEFAULT '{}'::jsonb; + +-- Create indexes for new columns +CREATE INDEX IF NOT EXISTS idx_debts_due_date ON debts(due_date); +CREATE INDEX IF NOT EXISTS idx_debts_metadata ON debts USING gin(metadata); + +-- Add comment for documentation +COMMENT ON COLUMN debts.description IS 'AI-extracted description of what the debt is for'; +COMMENT ON COLUMN debts.due_date IS 'Due date extracted from the email, if mentioned'; +COMMENT ON COLUMN debts.metadata IS 'Additional metadata including isDebtCollection flag and other AI-extracted information';