Chain APIs to scrape, summarise and score web pages

1
2

url ──► websitetotext ──► excerpt ─┬─► textsummarizer ──► summary
                                   └─► sentimentanalysis ──► tone

1
2
3
4

mkdir pipeline && cd pipeline
npm init -y
npm pkg set type=module
export APIVERVE_API_KEY=YOUR_API_KEY

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

const BASE = 'https://api.apiverve.com/v1';

export class VerveError extends Error {
  constructor(message, status, retryable) {
    super(message);
    this.status = status;
    this.retryable = retryable;
  }
}

export async function verve(api, body, { timeoutMs = 15000 } = {}) {
  const key = process.env.APIVERVE_API_KEY;
  if (!key) throw new Error('APIVERVE_API_KEY is not set');

  let res;
  try {
    res = await fetch(`${BASE}/${api}`, {
      method: 'POST',
      headers: { 'x-api-key': key, 'Content-Type': 'application/json' },
      body: JSON.stringify(body),
      signal: AbortSignal.timeout(timeoutMs),
    });
  } catch (err) {
    throw new VerveError(err.name === 'TimeoutError' ? 'timed out' : err.message, 0, true);
  }

  const json = await res.json().catch(() => null);
  if (!res.ok || json?.status !== 'ok') {
    const outOfCredits = res.status === 429 && res.headers.get('x-api-remaining-credits') === '0';
    const retryable = !outOfCredits && (res.status === 429 || res.status >= 500);
    throw new VerveError(outOfCredits ? 'out of credits' : json?.error || `HTTP ${res.status}`, res.status, retryable);
  }
  return json.data;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

import { verve } from './verve.js';

// Per-call credits from each API's catalog page.
const CREDITS = { websitetotext: 25, textsummarizer: 20, sentimentanalysis: 10 };
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function step(api, body, ledger, timeoutMs) {
  const started = Date.now();
  for (let attempt = 1; ; attempt++) {
    try {
      const data = await verve(api, body, { timeoutMs });
      ledger.push({ api, ok: true, attempt, ms: Date.now() - started, credits: CREDITS[api] });
      return data;
    } catch (err) {
      if (err.retryable && attempt < 2) {
        await sleep(1000);
        continue;
      }
      ledger.push({ api, ok: false, attempt, ms: Date.now() - started, credits: 0, error: err.message });
      throw err;
    }
  }
}

// Glue between step 1 and steps 2-3: clean, then cut under the summarizer's limit at a sentence end.
export function excerpt(text, max = 5000) {
  const clean = String(text).replace(/\[\d+\]/g, '').replace(/\s+/g, ' ').trim();
  if (clean.length <= max) return clean;
  const cut = clean.slice(0, max);
  const end = cut.lastIndexOf('. ');
  return end > max * 0.6 ? cut.slice(0, end + 1) : cut;
}

export async function analyse(url) {
  const ledger = [];
  const total = () => ledger.reduce((n, s) => n + s.credits, 0);

  let page;
  try {
    page = await step('websitetotext', { url }, ledger, 30000);
  } catch (err) {
    return { url, error: `could not read page: ${err.message}`, credits: total(), steps: ledger };
  }

  const text = excerpt(page.text ?? '');
  if (text.split(' ').length < 40) {
    return { url, error: 'page has too little text to analyse', credits: total(), steps: ledger };
  }

  const [summary, tone] = await Promise.allSettled([
    step('textsummarizer', { text, sentences: 3 }, ledger, 20000),
    step('sentimentanalysis', { text }, ledger, 15000),
  ]);

  return {
    url,
    title: page.title?.trim() || url,
    language: page.language ?? null,
    summary: summary.status === 'fulfilled' ? summary.value.summary : null,
    tone: tone.status === 'fulfilled'
      ? { perWord: tone.value.comparative, score: tone.value.sentiment, label: tone.value.sentimentText }
      : null,
    credits: total(),
    steps: ledger,
  };
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

import { analyse } from './pipeline.js';

async function pool(items, size, fn) {
  const out = [];
  for (let i = 0; i < items.length; i += size) {
    out.push(...(await Promise.all(items.slice(i, i + size).map(fn))));
  }
  return out;
}

const urls = process.argv.slice(2);
if (!urls.length) {
  console.error('Usage: node run.js <url> [url...]');
  process.exit(1);
}

const results = await pool(urls, 3, analyse);

for (const r of results) {
  console.log(`\n${r.title ?? r.url}`);
  if (r.error) {
    console.log(`  failed: ${r.error}`);
  } else {
    console.log(`  summary: ${r.summary ?? '(summary step failed)'}`);
    console.log(r.tone
      ? `  tone: ${r.tone.label} (score ${r.tone.score}, ${r.tone.perWord.toFixed(3)} per word)`
      : '  tone: (sentiment step failed)');
  }
  console.log(`  steps: ${r.steps.map((s) => `${s.api} ${s.ok ? 'ok' : 'FAILED'} ${s.ms}ms`).join(', ')}`);
  console.log(`  credits: ${r.credits}`);
}

console.log(`\nTotal credits: ${results.reduce((n, r) => n + r.credits, 0)} for ${urls.length} URL(s)`);

node run.js https://en.wikipedia.org/wiki/QR_code https://en.wikipedia.org/wiki/Exchange_rate

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

import os, re, sys
from concurrent.futures import ThreadPoolExecutor
import requests

BASE = "https://api.apiverve.com/v1"
CREDITS = {"websitetotext": 25, "textsummarizer": 20, "sentimentanalysis": 10}


def verve(api, body, timeout):
    res = requests.post(f"{BASE}/{api}", json=body,
                        headers={"x-api-key": os.environ["APIVERVE_API_KEY"]}, timeout=timeout)
    data = res.json()
    if res.status_code != 200 or data.get("status") != "ok":
        raise RuntimeError(data.get("error") or f"HTTP {res.status_code}")
    return data["data"]


def excerpt(text, limit=5000):
    clean = re.sub(r"\s+", " ", re.sub(r"\[\d+\]", "", text or "")).strip()
    if len(clean) <= limit:
        return clean
    cut = clean[:limit]
    end = cut.rfind(". ")
    return cut[: end + 1] if end > limit * 0.6 else cut


def analyse(url):
    credits = 0
    try:
        page = verve("websitetotext", {"url": url}, 30)
        credits += CREDITS["websitetotext"]
    except Exception as e:
        return {"url": url, "error": f"could not read page: {e}", "credits": credits}

    text = excerpt(page.get("text"))
    with ThreadPoolExecutor(max_workers=2) as ex:
        fut_sum = ex.submit(verve, "textsummarizer", {"text": text, "sentences": 3}, 20)
        fut_tone = ex.submit(verve, "sentimentanalysis", {"text": text}, 15)
        result = {"url": url, "title": (page.get("title") or "").strip() or url}
        for name, fut in (("textsummarizer", fut_sum), ("sentimentanalysis", fut_tone)):
            try:
                result[name] = fut.result()
                credits += CREDITS[name]
            except Exception as e:
                result[name] = None
                result[f"{name}_error"] = str(e)
    result["credits"] = credits
    return result


if __name__ == "__main__":
    for u in sys.argv[1:]:
        r = analyse(u)
        print(r["url"], "->", r.get("error") or (r["textsummarizer"] or {}).get("summary"))
        print("   tone:", (r.get("sentimentanalysis") or {}).get("comparative"), "| credits:", r["credits"])