Build an agent that reviews suspicious signups

1
2
3
4
5
6
7

const ACCOUNTS = [
  { id: 'acct_1001', normalizedEmail: 'janedoe@gmail.com', createdAt: '2026-08-02', status: 'trial_ended' },
];

export function findAccountsByNormalizedEmail(normalizedEmail) {
  return ACCOUNTS.filter((a) => a.normalizedEmail === normalizedEmail.toLowerCase());
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

import { runAgent } from './agent.js';
import { apiTools, executeApiTool } from './tools.js';
import { findAccountsByNormalizedEmail } from './db.js';

const SIGNAL_APIS = ['emailvalidator', 'vpndetector', 'iplookup'];

const LOCAL_TOOLS = [
  {
    name: 'find_accounts',
    description: 'Finds existing accounts in our database with the same normalized email address (use the normalizedEmail field from emailvalidator). Returns count and the accounts\' ids, creation dates and status. Read-only. Free.',
    schema: {
      type: 'object',
      properties: { normalizedEmail: { type: 'string' } },
      required: ['normalizedEmail'], additionalProperties: false,
    },
  },
  {
    name: 'submit_recommendation',
    description: 'Submits your recommendation for the human reviewer and ends the review. Every claim in summary must be supported by an evidence item naming the tool, the exact field and the value it returned.',
    schema: {
      type: 'object',
      properties: {
        recommendation: { type: 'string', enum: ['approve', 'request_verification', 'reject'] },
        confidence: { type: 'string', enum: ['low', 'medium', 'high'] },
        summary: { type: 'string', description: 'Three sentences at most, written for the reviewer.' },
        evidence: {
          type: 'array',
          items: {
            type: 'object',
            properties: {
              tool: { type: 'string' },
              field: { type: 'string' },
              value: { type: 'string', description: 'The value exactly as the tool returned it.' },
              why: { type: 'string' },
            },
            required: ['tool', 'field', 'value', 'why'], additionalProperties: false,
          },
        },
      },
      required: ['recommendation', 'confidence', 'summary', 'evidence'], additionalProperties: false,
    },
  },
];

const SYSTEM = `You investigate signups that automated checks flagged, and write a recommendation for a human reviewer.
You cannot approve, reject or contact anyone. A person decides.
Gather evidence with the tools, then call submit_recommendation once.
Weighting:
- isDisposable true, canReceiveMail false or isMxValid false: strong evidence against.
- normalizedEmail matching an existing account: strong evidence of a repeat trial. Say how many and when.
- is_vpn or is_datacenter true: weak on their own. Many real customers use VPNs and corporate networks.
- isRoleAccount true is not a fraud signal.
- Never treat a country or region as a reason on its own.
- A null or missing field means unknown, not false.
- When evidence is mixed or thin, recommend request_verification.`;

export async function triage(signup) {
  const observed = []; // every successful result, for checking evidence later

  async function execute(name, input) {
    if (SIGNAL_APIS.includes(name)) {
      const r = await executeApiTool(name, input);
      if (!r.isError) {
        try { observed.push({ tool: name, data: JSON.parse(r.content) }); } catch { /* truncated result */ }
      }
      return r;
    }
    if (name === 'find_accounts') {
      const accounts = findAccountsByNormalizedEmail(input.normalizedEmail);
      const data = { count: accounts.length, accounts };
      observed.push({ tool: name, data });
      return { content: JSON.stringify(data), isError: false };
    }
    if (name === 'submit_recommendation') return { content: 'Submitted.', isError: false, final: input };
    return { content: `Unknown tool ${name}.`, isError: true };
  }

  // Only what the investigation needs: no name, no payment details, no free-text profile.
  const prompt = `Flagged signup ${signup.signupId}
email: ${signup.email}
ip: ${signup.ip}
flagged because: ${signup.flaggedBecause}
signed up at: ${signup.createdAt}`;

  const run = await runAgent({
    system: SYSTEM, prompt, execute, maxTurns: 6,
    tools: [...apiTools(SIGNAL_APIS), ...LOCAL_TOOLS],
  });

  const recommendation = run.stopReason === 'final' ? run.final : null;
  const problems = recommendation ? checkEvidence(recommendation, observed) : [`Agent stopped without a recommendation (${run.stopReason}).`];
  return { recommendation, problems, observed };
}

export function checkEvidence(rec, observed) {
  const problems = [];
  if (rec.evidence.length === 0) problems.push('No evidence given.');
  for (const e of rec.evidence) {
    const hit = observed.find((o) => o.tool === e.tool && Object.hasOwn(o.data, e.field));
    if (!hit) problems.push(`No ${e.tool} result has a field "${e.field}".`);
    else if (String(hit.data[e.field]) !== String(e.value)) {
      problems.push(`${e.tool}.${e.field} was ${JSON.stringify(hit.data[e.field])}, not ${JSON.stringify(e.value)}.`);
    }
  }
  return problems;
}

npm install express@4

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

import express from 'express';
import { randomUUID } from 'node:crypto';
import { triage } from './triage.js';

const app = express();
app.use(express.json());
const reviews = new Map(); // use your database in production

// Called by your signup pipeline. Put all routes behind your admin authentication.
app.post('/flagged', async (req, res) => {
  const { signupId, email, ip, flaggedBecause } = req.body;
  if (!signupId || !email || !ip) return res.status(400).json({ error: 'signupId, email and ip are required' });
  const result = await triage({ signupId, email, ip, flaggedBecause, createdAt: new Date().toISOString() });
  const id = randomUUID();
  reviews.set(id, { id, signupId, status: 'pending', ...result });
  res.status(202).json({ id });
});

app.get('/reviews', (req, res) => {
  res.json([...reviews.values()].filter((r) => r.status === 'pending'));
});

app.post('/reviews/:id/decision', async (req, res) => {
  const review = reviews.get(req.params.id);
  if (!review || review.status !== 'pending') return res.status(404).json({ error: 'No pending review with that id' });
  const { decision, reviewer, note } = req.body;
  if (!['approve', 'request_verification', 'reject'].includes(decision) || !reviewer) {
    return res.status(400).json({ error: 'decision and reviewer are required' });
  }
  Object.assign(review, {
    status: 'decided', decision, reviewer, note,
    decidedAt: new Date().toISOString(),
    agreedWithAgent: review.recommendation ? decision === review.recommendation.recommendation : null,
  });
  await applyDecision(review.signupId, decision);
  res.json(review);
});

async function applyDecision(signupId, decision) {
  // Your existing account code: unlock, require verification, or close.
  console.log(`Applying ${decision} to ${signupId}`);
}

app.listen(3000, () => console.log('Review queue on http://localhost:3000'));

1
2
3
4

node server.js
curl -s -X POST localhost:3000/flagged -H 'Content-Type: application/json' \
  -d '{"signupId":"su_2002","email":"Jane.Doe+trial2@gmail.com","ip":"103.62.49.210","flaggedBecause":"datacenter IP"}'
curl -s localhost:3000/reviews