mirror of
https://github.com/FranP-code/inbox-negotiator.git
synced 2025-10-13 00:42:26 +00:00
Add AI parsing capabilities for debt information and update database schema
- Implemented AI-powered parsing for debt emails using Google's Gemini model. - Enhanced Postmark webhook to extract debt details including amount, vendor, description, due date, and legitimacy. - Updated database schema to include new columns for AI-extracted data. - Added environment variable requirements and updated package dependencies. - Created migration script for new database columns and indexes.
This commit is contained in:
10
.env.example
Normal file
10
.env.example
Normal file
@@ -0,0 +1,10 @@
|
||||
# Environment variables for Inbox Negotiator
|
||||
|
||||
# Supabase Configuration
|
||||
SUPABASE_URL=your_supabase_url_here
|
||||
SUPABASE_ANON_KEY=your_supabase_anon_key_here
|
||||
|
||||
# Google Generative AI API Key for Gemini model
|
||||
GOOGLE_GENERATIVE_AI_API_KEY=your_google_api_key_here
|
||||
|
||||
# Add these same variables to your actual .env file
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -22,3 +22,7 @@ pnpm-debug.log*
|
||||
|
||||
# jetbrains setting folder
|
||||
.idea/
|
||||
|
||||
# supabase files
|
||||
supabase/.branches
|
||||
supabase/.temp
|
||||
98
AI_PARSING_UPDATE.md
Normal file
98
AI_PARSING_UPDATE.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# AI-Enhanced Debt Parsing Update
|
||||
|
||||
## Changes Made
|
||||
|
||||
### 1. Postmark Endpoint Enhancement (`src/pages/api/postmark.ts`)
|
||||
|
||||
The Postmark webhook endpoint has been enhanced to use Vercel's AI package with Google's Gemini model for intelligent debt parsing.
|
||||
|
||||
#### Key Improvements:
|
||||
- **AI-Powered Parsing**: Replaced regex-based amount extraction with Gemini 1.5 Flash model
|
||||
- **Enhanced Data Extraction**: Now extracts:
|
||||
- Debt amount (more accurate than regex)
|
||||
- Vendor/creditor name
|
||||
- Description of what the debt is for
|
||||
- Due date (if mentioned)
|
||||
- Whether it's a legitimate debt collection notice
|
||||
|
||||
#### Fallback Mechanism:
|
||||
- If AI parsing fails or API key is not configured, falls back to original regex parsing
|
||||
- Ensures system reliability even if AI service is unavailable
|
||||
|
||||
### 2. Database Schema Updates
|
||||
|
||||
Added new columns to the `debts` table:
|
||||
- `description` (text) - AI-extracted description
|
||||
- `due_date` (timestamptz) - Extracted due date
|
||||
- `metadata` (jsonb) - Additional AI-extracted information
|
||||
|
||||
### 3. Dependencies Added
|
||||
|
||||
```bash
|
||||
npm install ai @ai-sdk/google
|
||||
```
|
||||
|
||||
### 4. Environment Variables Required
|
||||
|
||||
Add to your `.env` file:
|
||||
```bash
|
||||
GOOGLE_GENERATIVE_AI_API_KEY=your_google_api_key_here
|
||||
SUPABASE_URL=your_supabase_url
|
||||
SUPABASE_ANON_KEY=your_supabase_anon_key
|
||||
```
|
||||
|
||||
## How to Get Google API Key
|
||||
|
||||
1. Go to [Google AI Studio](https://aistudio.google.com/)
|
||||
2. Create a new API key
|
||||
3. Add it to your environment variables as `GOOGLE_GENERATIVE_AI_API_KEY`
|
||||
|
||||
## Testing the Changes
|
||||
|
||||
### 1. Test Email Processing
|
||||
Send a test email to your Postmark webhook endpoint with debt-related content like:
|
||||
```
|
||||
Subject: Outstanding Balance Notice
|
||||
|
||||
Dear Customer,
|
||||
|
||||
You have an outstanding balance of $150.00 for your account with XYZ Collections.
|
||||
This amount is due by January 15, 2025.
|
||||
|
||||
Please contact us to arrange payment.
|
||||
```
|
||||
|
||||
### 2. Expected Database Entry
|
||||
The system should now create a debt record with:
|
||||
- `amount`: 150.00
|
||||
- `vendor`: sender's email
|
||||
- `description`: AI-generated description
|
||||
- `due_date`: 2025-01-15 (if extracted)
|
||||
- `metadata`: JSON with isDebtCollection flag and other details
|
||||
|
||||
### 3. Migration Application
|
||||
If using local Supabase, apply the migration:
|
||||
```bash
|
||||
supabase db reset
|
||||
# or
|
||||
supabase migration up
|
||||
```
|
||||
|
||||
For production, apply the migration found in:
|
||||
`supabase/migrations/20250607000500_add_ai_parsing_columns.sql`
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **More Accurate Parsing**: AI can understand context better than regex
|
||||
2. **Richer Data**: Extracts more information from emails
|
||||
3. **Better Classification**: Determines if email is actually a debt collection notice
|
||||
4. **Future-Proof**: Can be enhanced with more sophisticated AI models
|
||||
5. **Reliable Fallback**: Still works if AI service is unavailable
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Set up Google API key
|
||||
2. Apply database migration
|
||||
3. Test with sample debt collection emails
|
||||
4. Monitor logs for AI parsing accuracy
|
||||
5. Consider training on domain-specific examples for better accuracy
|
||||
243
package-lock.json
generated
243
package-lock.json
generated
@@ -8,8 +8,10 @@
|
||||
"name": "@example/basics",
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"@ai-sdk/google": "^1.2.19",
|
||||
"@astrojs/react": "^3.6.2",
|
||||
"@astrojs/tailwind": "^5.1.1",
|
||||
"@google/generative-ai": "^0.24.1",
|
||||
"@hookform/resolvers": "^3.9.0",
|
||||
"@radix-ui/react-accordion": "^1.2.0",
|
||||
"@radix-ui/react-alert-dialog": "^1.1.1",
|
||||
@@ -41,6 +43,7 @@
|
||||
"@supabase/supabase-js": "^2.39.0",
|
||||
"@types/react": "^18.3.10",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"ai": "^4.3.16",
|
||||
"astro": "^4.15.9",
|
||||
"class-variance-authority": "^0.7.0",
|
||||
"clsx": "^2.1.1",
|
||||
@@ -64,6 +67,92 @@
|
||||
"zod": "^3.23.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/google": {
|
||||
"version": "1.2.19",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-1.2.19.tgz",
|
||||
"integrity": "sha512-Xgl6eftIRQ4srUdCzxM112JuewVMij5q4JLcNmHcB68Bxn9dpr3MVUSPlJwmameuiQuISIA8lMB+iRiRbFsaqA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "1.1.3",
|
||||
"@ai-sdk/provider-utils": "2.2.8"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/provider": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz",
|
||||
"integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"json-schema": "^0.4.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/provider-utils": {
|
||||
"version": "2.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz",
|
||||
"integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "1.1.3",
|
||||
"nanoid": "^3.3.8",
|
||||
"secure-json-parse": "^2.7.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.23.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/react": {
|
||||
"version": "1.2.12",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz",
|
||||
"integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider-utils": "2.2.8",
|
||||
"@ai-sdk/ui-utils": "1.2.11",
|
||||
"swr": "^2.2.5",
|
||||
"throttleit": "2.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^18 || ^19 || ^19.0.0-rc",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"zod": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/ui-utils": {
|
||||
"version": "1.2.11",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz",
|
||||
"integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "1.1.3",
|
||||
"@ai-sdk/provider-utils": "2.2.8",
|
||||
"zod-to-json-schema": "^3.24.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.23.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@alloc/quick-lru": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",
|
||||
@@ -886,6 +975,15 @@
|
||||
"resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.8.tgz",
|
||||
"integrity": "sha512-kym7SodPp8/wloecOpcmSnWJsK7M0E5Wg8UcFA+uO4B9s5d0ywXOEro/8HM9x0rW+TljRzul/14UYz3TleT3ig=="
|
||||
},
|
||||
"node_modules/@google/generative-ai": {
|
||||
"version": "0.24.1",
|
||||
"resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz",
|
||||
"integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@hookform/resolvers": {
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/@hookform/resolvers/-/resolvers-3.9.0.tgz",
|
||||
@@ -1375,6 +1473,15 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/api": {
|
||||
"version": "1.9.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
|
||||
"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oslojs/encoding": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@oslojs/encoding/-/encoding-1.1.0.tgz",
|
||||
@@ -3164,6 +3271,12 @@
|
||||
"@types/ms": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/diff-match-patch": {
|
||||
"version": "1.0.36",
|
||||
"resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz",
|
||||
"integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
|
||||
@@ -3283,6 +3396,32 @@
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ai": {
|
||||
"version": "4.3.16",
|
||||
"resolved": "https://registry.npmjs.org/ai/-/ai-4.3.16.tgz",
|
||||
"integrity": "sha512-KUDwlThJ5tr2Vw0A1ZkbDKNME3wzWhuVfAOwIvFUzl1TPVDFAXDFTXio3p+jaKneB+dKNCvFFlolYmmgHttG1g==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "1.1.3",
|
||||
"@ai-sdk/provider-utils": "2.2.8",
|
||||
"@ai-sdk/react": "1.2.12",
|
||||
"@ai-sdk/ui-utils": "1.2.11",
|
||||
"@opentelemetry/api": "1.9.0",
|
||||
"jsondiffpatch": "0.6.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^18 || ^19 || ^19.0.0-rc",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"react": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-align": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz",
|
||||
@@ -4534,6 +4673,12 @@
|
||||
"node": ">=0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/diff-match-patch": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz",
|
||||
"integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/dlv": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
|
||||
@@ -5390,6 +5535,12 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/json-schema": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz",
|
||||
"integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==",
|
||||
"license": "(AFL-2.1 OR BSD-3-Clause)"
|
||||
},
|
||||
"node_modules/json5": {
|
||||
"version": "2.2.3",
|
||||
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
|
||||
@@ -5401,6 +5552,35 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/jsondiffpatch": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.6.0.tgz",
|
||||
"integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/diff-match-patch": "^1.0.36",
|
||||
"chalk": "^5.3.0",
|
||||
"diff-match-patch": "^1.0.5"
|
||||
},
|
||||
"bin": {
|
||||
"jsondiffpatch": "bin/jsondiffpatch.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.0.0 || >=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/jsondiffpatch/node_modules/chalk": {
|
||||
"version": "5.4.1",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz",
|
||||
"integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^12.17.0 || ^14.13 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
|
||||
@@ -6425,15 +6605,16 @@
|
||||
}
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.7",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz",
|
||||
"integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==",
|
||||
"version": "3.3.11",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
|
||||
"integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/ai"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"nanoid": "bin/nanoid.cjs"
|
||||
},
|
||||
@@ -7509,6 +7690,12 @@
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/secure-json-parse": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
|
||||
"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "6.3.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
|
||||
@@ -7828,6 +8015,19 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/swr": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/swr/-/swr-2.3.3.tgz",
|
||||
"integrity": "sha512-dshNvs3ExOqtZ6kJBaAsabhPdHyeY4P2cKwRCniDVifBMoG/SVI7tfLWqPXriVspf2Rg4tPzXJTnwaihIeFw2A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"dequal": "^2.0.3",
|
||||
"use-sync-external-store": "^1.4.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tailwind-merge": {
|
||||
"version": "2.5.3",
|
||||
"resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.5.3.tgz",
|
||||
@@ -7919,6 +8119,18 @@
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/throttleit": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz",
|
||||
"integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/tiny-invariant": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz",
|
||||
@@ -8238,6 +8450,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/use-sync-external-store": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.5.0.tgz",
|
||||
"integrity": "sha512-Rb46I4cGGVBmjamjphe8L/UnvJD+uPPtTkNvX5mZgqdbavhI4EbgIWJiIHXJ8bc/i9EQGPRh4DwEURJ552Do0A==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
@@ -8635,19 +8856,21 @@
|
||||
}
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "3.23.8",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz",
|
||||
"integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==",
|
||||
"version": "3.25.56",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.56.tgz",
|
||||
"integrity": "sha512-rd6eEF3BTNvQnR2e2wwolfTmUTnp70aUTqr0oaGbHifzC3BKJsoV+Gat8vxUMR1hwOKBs6El+qWehrHbCpW6SQ==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
},
|
||||
"node_modules/zod-to-json-schema": {
|
||||
"version": "3.23.3",
|
||||
"resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.3.tgz",
|
||||
"integrity": "sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog==",
|
||||
"version": "3.24.5",
|
||||
"resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz",
|
||||
"integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==",
|
||||
"license": "ISC",
|
||||
"peerDependencies": {
|
||||
"zod": "^3.23.3"
|
||||
"zod": "^3.24.1"
|
||||
}
|
||||
},
|
||||
"node_modules/zod-to-ts": {
|
||||
|
||||
@@ -11,8 +11,10 @@
|
||||
"astro": "astro"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/google": "^1.2.19",
|
||||
"@astrojs/react": "^3.6.2",
|
||||
"@astrojs/tailwind": "^5.1.1",
|
||||
"@google/generative-ai": "^0.24.1",
|
||||
"@hookform/resolvers": "^3.9.0",
|
||||
"@radix-ui/react-accordion": "^1.2.0",
|
||||
"@radix-ui/react-alert-dialog": "^1.1.1",
|
||||
@@ -44,6 +46,7 @@
|
||||
"@supabase/supabase-js": "^2.39.0",
|
||||
"@types/react": "^18.3.10",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"ai": "^4.3.16",
|
||||
"astro": "^4.15.9",
|
||||
"class-variance-authority": "^0.7.0",
|
||||
"clsx": "^2.1.1",
|
||||
|
||||
@@ -1,98 +1,176 @@
|
||||
import type { APIRoute } from 'astro';
|
||||
import { supabase } from '../../lib/supabase';
|
||||
import type { APIRoute } from "astro";
|
||||
import { supabase } from "../../lib/supabase";
|
||||
import { generateObject } from "ai";
|
||||
import {
|
||||
createGoogleGenerativeAI,
|
||||
google,
|
||||
type GoogleGenerativeAIProviderOptions,
|
||||
} from "@ai-sdk/google";
|
||||
import { z } from "zod";
|
||||
|
||||
// Schema for debt information extraction
|
||||
const debtSchema = z.object({
|
||||
amount: z.number().min(0).describe("The debt amount in dollars"),
|
||||
vendor: z.string().describe("The name or identifier of the vendor/creditor"),
|
||||
description: z.string().describe("Brief description of what the debt is for"),
|
||||
dueDate: z.string().optional().describe("Due date if mentioned (ISO format)"),
|
||||
isDebtCollection: z
|
||||
.boolean()
|
||||
.describe("Whether this appears to be a debt collection notice"),
|
||||
});
|
||||
|
||||
// Function to parse debt information using AI
|
||||
async function parseDebtWithAI(emailText: string, fromEmail: string) {
|
||||
try {
|
||||
// Check if Google API key is available
|
||||
const googleApiKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY;
|
||||
if (!googleApiKey) {
|
||||
console.warn(
|
||||
"Google API key not configured, falling back to regex parsing"
|
||||
);
|
||||
throw new Error("No Google API key configured");
|
||||
}
|
||||
|
||||
const result = await generateObject({
|
||||
model: createGoogleGenerativeAI({
|
||||
apiKey: googleApiKey,
|
||||
})("gemini-2.5-flash-preview-04-17"),
|
||||
system: `You are an expert at analyzing debt collection and billing emails.
|
||||
Extract key debt information from the email content.
|
||||
Look for monetary amounts, creditor information, what the debt is for, and due dates.
|
||||
If this doesn't appear to be a legitimate debt or billing notice, set amount to 0.
|
||||
Be very accurate with amounts - look for dollar signs and numbers carefully.`,
|
||||
prompt: `Parse this email for debt information:
|
||||
|
||||
From: ${fromEmail}
|
||||
Content: ${emailText}`,
|
||||
schema: debtSchema,
|
||||
});
|
||||
|
||||
return result.object;
|
||||
} catch (error) {
|
||||
console.error("AI parsing error:", error);
|
||||
// Fallback to regex if AI fails
|
||||
const amountMatch = emailText.match(/\$(\d+(?:,\d{3})*(?:\.\d{2})?)/);
|
||||
return {
|
||||
amount: amountMatch ? parseFloat(amountMatch[1].replace(/,/g, "")) : 0,
|
||||
vendor: fromEmail || "unknown",
|
||||
description: "Failed to parse with AI - using regex fallback",
|
||||
isDebtCollection: amountMatch ? true : false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const POST: APIRoute = async ({ request }) => {
|
||||
try {
|
||||
const data = await request.json();
|
||||
|
||||
// Check for opt-out keywords
|
||||
const optOutKeywords = ['STOP', 'UNSUBSCRIBE', 'OPT-OUT', 'REMOVE'];
|
||||
const textBody = data.TextBody || '';
|
||||
const hasOptOut = optOutKeywords.some(keyword =>
|
||||
const optOutKeywords = ["STOP", "UNSUBSCRIBE", "OPT-OUT", "REMOVE"];
|
||||
const textBody = data.TextBody || "";
|
||||
const fromEmail = data.FromFull?.Email || "unknown";
|
||||
|
||||
const hasOptOut = optOutKeywords.some((keyword) =>
|
||||
textBody.toUpperCase().includes(keyword)
|
||||
);
|
||||
|
||||
if (hasOptOut) {
|
||||
// Log opt-out and don't process further
|
||||
const { error } = await supabase.from('debts').insert({
|
||||
vendor: data.FromFull?.Email || 'unknown',
|
||||
const { error } = await supabase.from("debts").insert({
|
||||
vendor: fromEmail,
|
||||
amount: 0,
|
||||
raw_email: textBody,
|
||||
status: 'opted_out'
|
||||
status: "opted_out",
|
||||
});
|
||||
|
||||
if (error) {
|
||||
console.error('Error logging opt-out:', error);
|
||||
console.error("Error logging opt-out:", error);
|
||||
return new Response(JSON.stringify({ error: error.message }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
return new Response('Opt-out processed', { status: 200 });
|
||||
return new Response("Opt-out processed", { status: 200 });
|
||||
}
|
||||
|
||||
// Extract debt amount using regex
|
||||
const amountMatch = textBody.match(/\$(\d+(?:,\d{3})*(?:\.\d{2})?)/);
|
||||
const amount = amountMatch ? parseFloat(amountMatch[1].replace(/,/g, '')) : 0;
|
||||
// Parse debt information using AI
|
||||
const debtInfo = await parseDebtWithAI(textBody, fromEmail);
|
||||
|
||||
// Insert debt record
|
||||
// Insert debt record with AI-extracted information
|
||||
const { data: insertedDebt, error: insertError } = await supabase
|
||||
.from('debts')
|
||||
.from("debts")
|
||||
.insert({
|
||||
vendor: data.FromFull?.Email || 'unknown',
|
||||
amount: amount,
|
||||
vendor: debtInfo.vendor,
|
||||
amount: debtInfo.amount,
|
||||
raw_email: textBody,
|
||||
status: 'received'
|
||||
status: "received",
|
||||
description: debtInfo.description,
|
||||
due_date: debtInfo.dueDate,
|
||||
metadata: {
|
||||
isDebtCollection: debtInfo.isDebtCollection,
|
||||
subject: data.Subject,
|
||||
fromEmail: fromEmail,
|
||||
},
|
||||
})
|
||||
.select()
|
||||
.single();
|
||||
|
||||
if (insertError) {
|
||||
console.error('Error inserting debt:', insertError);
|
||||
console.error("Error inserting debt:", insertError);
|
||||
return new Response(JSON.stringify({ error: insertError.message }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
// Log the email receipt
|
||||
await supabase.from('audit_logs').insert({
|
||||
await supabase.from("audit_logs").insert({
|
||||
debt_id: insertedDebt.id,
|
||||
action: 'email_received',
|
||||
action: "email_received",
|
||||
details: {
|
||||
vendor: data.FromFull?.Email,
|
||||
amount: amount,
|
||||
subject: data.Subject
|
||||
}
|
||||
vendor: debtInfo.vendor,
|
||||
amount: debtInfo.amount,
|
||||
subject: data.Subject,
|
||||
aiParsed: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Trigger negotiation function
|
||||
if (amount > 0) {
|
||||
const negotiateUrl = `${import.meta.env.SUPABASE_URL}/functions/v1/negotiate`;
|
||||
// Trigger negotiation function if this is a legitimate debt
|
||||
if (debtInfo.amount > 0 && debtInfo.isDebtCollection) {
|
||||
// Access environment variables through Astro runtime
|
||||
const supabaseUrl = process.env.SUPABASE_URL;
|
||||
const supabaseAnonKey = process.env.SUPABASE_ANON_KEY;
|
||||
|
||||
if (supabaseUrl && supabaseAnonKey) {
|
||||
const negotiateUrl = `${supabaseUrl}/functions/v1/negotiate`;
|
||||
|
||||
try {
|
||||
await fetch(negotiateUrl, {
|
||||
method: 'POST',
|
||||
method: "POST",
|
||||
headers: {
|
||||
'Authorization': `Bearer ${import.meta.env.SUPABASE_ANON_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${supabaseAnonKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ record: insertedDebt })
|
||||
body: JSON.stringify({ record: insertedDebt }),
|
||||
});
|
||||
} catch (negotiateError) {
|
||||
console.error('Error triggering negotiation:', negotiateError);
|
||||
console.error("Error triggering negotiation:", negotiateError);
|
||||
// Don't fail the webhook if negotiation fails
|
||||
}
|
||||
} else {
|
||||
console.warn(
|
||||
"Supabase environment variables not configured for negotiation trigger"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return new Response('OK', { status: 200 });
|
||||
|
||||
return new Response("OK", { status: 200 });
|
||||
} catch (error) {
|
||||
console.error('Postmark webhook error:', error);
|
||||
return new Response(JSON.stringify({ error: 'Internal server error' }), {
|
||||
console.error("Postmark webhook error:", error);
|
||||
return new Response(JSON.stringify({ error: "Internal server error" }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Add new columns for AI-enhanced debt parsing
|
||||
-- Migration for improved debt information storage
|
||||
|
||||
-- Add new columns to debts table
|
||||
ALTER TABLE debts ADD COLUMN IF NOT EXISTS description text;
|
||||
ALTER TABLE debts ADD COLUMN IF NOT EXISTS due_date timestamptz;
|
||||
ALTER TABLE debts ADD COLUMN IF NOT EXISTS metadata jsonb DEFAULT '{}'::jsonb;
|
||||
|
||||
-- Create indexes for new columns
|
||||
CREATE INDEX IF NOT EXISTS idx_debts_due_date ON debts(due_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_debts_metadata ON debts USING gin(metadata);
|
||||
|
||||
-- Add comment for documentation
|
||||
COMMENT ON COLUMN debts.description IS 'AI-extracted description of what the debt is for';
|
||||
COMMENT ON COLUMN debts.due_date IS 'Due date extracted from the email, if mentioned';
|
||||
COMMENT ON COLUMN debts.metadata IS 'Additional metadata including isDebtCollection flag and other AI-extracted information';
|
||||
Reference in New Issue
Block a user