|
| 1 | +#!/usr/bin/env node |
| 2 | +// New internal displacement events from IDMC's Internal Displacement Updates |
| 3 | +// (IDU), read from the per-country CSVs IDMC publishes on HDX |
| 4 | +// (<iso3>-idmc-idu-events). The Helix API carries the same rows but needs a |
| 5 | +// client_id IDMC has not issued (#9014). |
| 6 | +// |
| 7 | +// IDMC enters a figure a median of 23 days after the displacement (p90 65 days, |
| 8 | +// measured 2026-10-20), so the newest week is always thin. Surges are judged on |
| 9 | +// a 30-day window against the monthly mean of the 150 days before it, and |
| 10 | +// newlyReported7d counts figures by publication date instead. |
| 11 | + |
| 12 | +import Papa from 'papaparse'; |
| 13 | + |
| 14 | +import { loadEnvFile, CHROME_UA, runSeed, withRetry } from './_seed-utils.mjs'; |
| 15 | +import { DAY_MIN, tokensToContentMeta } from './_content-age-helpers.mjs'; |
| 16 | +import { getOptionalUpstashCreds, upstashCommand } from './_upstash-rest.mjs'; |
| 17 | + |
| 18 | +loadEnvFile(import.meta.url); |
| 19 | + |
| 20 | +export const IDU_CANONICAL_KEY = 'displacement:idmc-idu:v1'; |
| 21 | +export const IDU_ACTIVATION_KEY = 'seed-activated:displacement:idmc-idu'; |
| 22 | +const HDX_SEARCH_URL = 'https://data.humdata.org/api/3/action/package_search'; |
| 23 | +const HDX_ORGANIZATION = 'international-displacement-monitoring-centre-idmc'; |
| 24 | +const DATASET_NAME = /^([a-z]{3})-idmc-idu-events$/; |
| 25 | +const SEARCH_ROWS = 1000; |
| 26 | +const CACHE_TTL_SECONDS = 3 * 24 * 60 * 60; |
| 27 | +const FETCH_CONCURRENCY = 6; |
| 28 | +const REQUEST_TIMEOUT_MS = 60_000; |
| 29 | +// Largest file was 4.3 MB (PER) on 2026-10-20. |
| 30 | +const MAX_CSV_BYTES = 32 * 1024 * 1024; |
| 31 | +const DAY_MS = 24 * 60 * 60 * 1000; |
| 32 | +export const WINDOW_DAYS = 180; |
| 33 | +export const RECENT_DAYS = 30; |
| 34 | +export const NEWLY_REPORTED_DAYS = 7; |
| 35 | +export const SURGE_MIN_DISPLACED = 5000; |
| 36 | +export const SURGE_RATIO = 3; |
| 37 | +// One dataset that HDX lists but will not serve must not hide every other |
| 38 | +// country; more than this many is an outage, not a bad file. |
| 39 | +export const MAX_MISSING_COUNTRIES = 3; |
| 40 | +export const MIN_COUNTRIES = 30; |
| 41 | + |
| 42 | +const REQUIRED_COLUMNS = [ |
| 43 | + 'id', 'iso3', 'country', 'latitude', 'longitude', 'role', 'displacement_type', 'figure', |
| 44 | + 'displacement_date', 'event_id', 'event_name', 'created_at', |
| 45 | +]; |
| 46 | + |
| 47 | +async function getWithTimeout(url) { |
| 48 | + return withRetry(async () => { |
| 49 | + const resp = await fetch(url, { |
| 50 | + headers: { Accept: 'application/json, text/csv', 'User-Agent': CHROME_UA }, |
| 51 | + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), |
| 52 | + }); |
| 53 | + if (!resp.ok) { |
| 54 | + const err = new Error(`HDX ${url}: HTTP ${resp.status}`); |
| 55 | + // A missing or forbidden resource stays that way within one run. |
| 56 | + err.nonRetryable = resp.status >= 400 && resp.status < 500 && resp.status !== 429; |
| 57 | + throw err; |
| 58 | + } |
| 59 | + return resp; |
| 60 | + }, 2, 2000); |
| 61 | +} |
| 62 | + |
| 63 | +/** Lists the IDU event datasets and their CSV resource. */ |
| 64 | +export async function listIduDatasets() { |
| 65 | + const query = new URLSearchParams({ fq: `organization:${HDX_ORGANIZATION}`, rows: String(SEARCH_ROWS) }); |
| 66 | + const body = await (await getWithTimeout(`${HDX_SEARCH_URL}?${query}`)).json(); |
| 67 | + if (body?.success !== true) throw new Error('HDX package_search returned success=false'); |
| 68 | + const { count, results } = body.result ?? {}; |
| 69 | + if (!Array.isArray(results)) throw new Error('HDX package_search returned no results array'); |
| 70 | + if (Number(count) > results.length) throw new Error(`HDX package_search truncated: ${results.length} of ${count} datasets`); |
| 71 | + |
| 72 | + const datasets = []; |
| 73 | + for (const pkg of results) { |
| 74 | + const match = DATASET_NAME.exec(pkg?.name ?? ''); |
| 75 | + if (!match) continue; |
| 76 | + const csv = (pkg.resources ?? []).find((r) => String(r?.format ?? '').toUpperCase() === 'CSV' && r?.url); |
| 77 | + datasets.push({ countryCode: match[1].toUpperCase(), url: csv?.url ?? null }); |
| 78 | + } |
| 79 | + return datasets; |
| 80 | +} |
| 81 | + |
| 82 | +async function fetchCsv(url) { |
| 83 | + const resp = await getWithTimeout(url); |
| 84 | + const length = Number(resp.headers.get('content-length')); |
| 85 | + if (Number.isFinite(length) && length > MAX_CSV_BYTES) throw new Error(`HDX ${url}: ${length} bytes exceeds ${MAX_CSV_BYTES}`); |
| 86 | + const text = await resp.text(); |
| 87 | + if (text.length > MAX_CSV_BYTES) throw new Error(`HDX ${url}: body exceeds ${MAX_CSV_BYTES} bytes`); |
| 88 | + return text; |
| 89 | +} |
| 90 | + |
| 91 | +export function parseIduCsv(text) { |
| 92 | + const parsed = Papa.parse(String(text ?? ''), { |
| 93 | + header: true, |
| 94 | + skipEmptyLines: 'greedy', |
| 95 | + transformHeader: (header) => header.replace(/^\ufeff/, '').trim(), |
| 96 | + }); |
| 97 | + const fields = new Set(parsed.meta?.fields ?? []); |
| 98 | + const missing = REQUIRED_COLUMNS.filter((field) => !fields.has(field)); |
| 99 | + if (missing.length > 0) throw new Error(`IDU CSV is missing columns: ${missing.join(', ')}`); |
| 100 | + if (parsed.errors?.length > 0) throw new Error(`IDU CSV is invalid: ${parsed.errors[0].message}`); |
| 101 | + return parsed.data; |
| 102 | +} |
| 103 | + |
| 104 | +const isoDate = (value) => (/^\d{4}-\d{2}-\d{2}/.test(value ?? '') ? value.slice(0, 10) : ''); |
| 105 | +const daysAgo = (nowMs, days) => new Date(nowMs - days * DAY_MS).toISOString().slice(0, 10); |
| 106 | +const text = (value) => String(value ?? '').trim(); |
| 107 | + |
| 108 | +/** |
| 109 | + * Normalizes IDU rows into events inside the window. Only "Recommended figure" |
| 110 | + * rows count: "Triangulation" rows cross-check the same displacement. |
| 111 | + */ |
| 112 | +export function buildEvents(rows, { nowMs = Date.now() } = {}) { |
| 113 | + const from = daysAgo(nowMs, WINDOW_DAYS); |
| 114 | + const today = daysAgo(nowMs, 0); |
| 115 | + const events = []; |
| 116 | + for (const row of rows) { |
| 117 | + if (text(row?.role) !== 'Recommended figure') continue; |
| 118 | + const displacementDate = isoDate(row.displacement_date); |
| 119 | + if (!displacementDate || displacementDate < from || displacementDate > today) continue; |
| 120 | + const figure = Number(row.figure); |
| 121 | + const latitude = Number(row.latitude); |
| 122 | + const longitude = Number(row.longitude); |
| 123 | + const countryCode = text(row.iso3).toUpperCase(); |
| 124 | + if (!Number.isFinite(figure) || figure <= 0 || !/^[A-Z]{3}$/.test(countryCode)) continue; |
| 125 | + const hasPoint = text(row.latitude) !== '' && text(row.longitude) !== '' |
| 126 | + && Math.abs(latitude) <= 90 && Math.abs(longitude) <= 180; |
| 127 | + events.push({ |
| 128 | + id: text(row.id), |
| 129 | + countryCode, |
| 130 | + countryName: text(row.country) || countryCode, |
| 131 | + displacementType: text(row.displacement_type), |
| 132 | + figure, |
| 133 | + qualifier: text(row.qualifier), |
| 134 | + displacementDate, |
| 135 | + startDate: isoDate(row.displacement_start_date), |
| 136 | + endDate: isoDate(row.displacement_end_date), |
| 137 | + eventId: text(row.event_id), |
| 138 | + eventName: text(row.event_name), |
| 139 | + category: text(row.category), |
| 140 | + hazardType: text(row.type), |
| 141 | + locationName: text(row.locations_name), |
| 142 | + locationAccuracy: text(row.locations_accuracy), |
| 143 | + sources: text(row.sources), |
| 144 | + createdAt: isoDate(row.created_at), |
| 145 | + ...(hasPoint ? { location: { latitude, longitude } } : {}), |
| 146 | + }); |
| 147 | + } |
| 148 | + return events.sort((a, b) => b.displacementDate.localeCompare(a.displacementDate) || b.figure - a.figure); |
| 149 | +} |
| 150 | + |
| 151 | +/** Per-country totals and the surge flag. */ |
| 152 | +export function summarizeCountries(events, { nowMs = Date.now() } = {}) { |
| 153 | + const recentFrom = daysAgo(nowMs, RECENT_DAYS); |
| 154 | + const reportedFrom = daysAgo(nowMs, NEWLY_REPORTED_DAYS); |
| 155 | + const baselineMonths = (WINDOW_DAYS - RECENT_DAYS) / 30; |
| 156 | + const byCountry = new Map(); |
| 157 | + for (const ev of events) { |
| 158 | + let c = byCountry.get(ev.countryCode); |
| 159 | + if (!c) { |
| 160 | + c = { |
| 161 | + countryCode: ev.countryCode, |
| 162 | + countryName: ev.countryName, |
| 163 | + displaced30d: 0, |
| 164 | + conflict30d: 0, |
| 165 | + disaster30d: 0, |
| 166 | + events30d: 0, |
| 167 | + baselineTotal: 0, |
| 168 | + newlyReported7d: 0, |
| 169 | + latestDisplacementDate: '', |
| 170 | + }; |
| 171 | + byCountry.set(ev.countryCode, c); |
| 172 | + } |
| 173 | + if (ev.displacementDate >= recentFrom) { |
| 174 | + c.displaced30d += ev.figure; |
| 175 | + c.events30d += 1; |
| 176 | + if (ev.displacementType === 'Conflict') c.conflict30d += ev.figure; |
| 177 | + else if (ev.displacementType === 'Disaster') c.disaster30d += ev.figure; |
| 178 | + } else { |
| 179 | + c.baselineTotal += ev.figure; |
| 180 | + } |
| 181 | + if (ev.createdAt >= reportedFrom) c.newlyReported7d += ev.figure; |
| 182 | + if (ev.displacementDate > c.latestDisplacementDate) c.latestDisplacementDate = ev.displacementDate; |
| 183 | + } |
| 184 | + return [...byCountry.values()] |
| 185 | + .map(({ baselineTotal, ...c }) => { |
| 186 | + const baseline = baselineTotal / baselineMonths; |
| 187 | + return { |
| 188 | + ...c, |
| 189 | + baselineMonthly: Math.round(baseline), |
| 190 | + surge: c.displaced30d >= SURGE_MIN_DISPLACED && c.displaced30d >= SURGE_RATIO * baseline, |
| 191 | + }; |
| 192 | + }) |
| 193 | + .sort((a, b) => b.displaced30d - a.displaced30d || b.baselineMonthly - a.baselineMonthly); |
| 194 | +} |
| 195 | + |
| 196 | +export async function fetchIduDisplacement({ nowMs = Date.now() } = {}) { |
| 197 | + const datasets = await listIduDatasets(); |
| 198 | + if (datasets.length === 0) throw new Error('HDX lists no IDMC IDU event datasets'); |
| 199 | + |
| 200 | + const rows = []; |
| 201 | + const missingCountries = []; |
| 202 | + const queue = [...datasets]; |
| 203 | + await Promise.all(Array.from({ length: FETCH_CONCURRENCY }, async () => { |
| 204 | + while (queue.length > 0) { |
| 205 | + const { countryCode, url } = queue.shift(); |
| 206 | + try { |
| 207 | + if (!url) throw new Error('no CSV resource'); |
| 208 | + rows.push(...parseIduCsv(await fetchCsv(url))); |
| 209 | + } catch (err) { |
| 210 | + missingCountries.push(countryCode); |
| 211 | + console.warn(` WARN: IDU ${countryCode} skipped: ${err?.message || err}`); |
| 212 | + } |
| 213 | + } |
| 214 | + })); |
| 215 | + if (missingCountries.length > MAX_MISSING_COUNTRIES) { |
| 216 | + throw new Error(`IDU: ${missingCountries.length} of ${datasets.length} country files failed (${missingCountries.sort().join(', ')})`); |
| 217 | + } |
| 218 | + |
| 219 | + const events = buildEvents(rows, { nowMs }); |
| 220 | + const countries = summarizeCountries(events, { nowMs }); |
| 221 | + const surges = countries.filter((c) => c.surge).map((c) => c.countryCode); |
| 222 | + console.log(` IDU: ${datasets.length} datasets, ${rows.length} rows, ${events.length} events in ${countries.length} countries since ${daysAgo(nowMs, WINDOW_DAYS)}; surges: ${surges.join(', ') || 'none'}`); |
| 223 | + return { |
| 224 | + source: 'IDMC Internal Displacement Updates (via HDX)', |
| 225 | + windowFrom: daysAgo(nowMs, WINDOW_DAYS), |
| 226 | + missingCountries: missingCountries.sort(), |
| 227 | + countries, |
| 228 | + events, |
| 229 | + }; |
| 230 | +} |
| 231 | + |
| 232 | +export function validate(data) { |
| 233 | + return Array.isArray(data?.events) && Array.isArray(data?.countries) && data.countries.length >= MIN_COUNTRIES; |
| 234 | +} |
| 235 | + |
| 236 | +export function declareRecords(data) { |
| 237 | + return data?.events?.length ?? 0; |
| 238 | +} |
| 239 | + |
| 240 | +// Dated by publication, not displacement, so a quiet month on the ground does |
| 241 | +// not read as a stale feed. |
| 242 | +export function contentMeta(data, nowMs = Date.now()) { |
| 243 | + return tokensToContentMeta((data?.events ?? []).map((ev) => ev.createdAt), nowMs); |
| 244 | +} |
| 245 | + |
| 246 | +// Durable marker for /api/health, written only after a successful publish. |
| 247 | +async function markActivated() { |
| 248 | + try { |
| 249 | + const creds = getOptionalUpstashCreds(); |
| 250 | + if (!creds) return; |
| 251 | + await upstashCommand(creds, ['SET', IDU_ACTIVATION_KEY, '1']); |
| 252 | + } catch (err) { |
| 253 | + console.warn(` WARN: IDU activation marker write failed: ${err?.message || err}`); |
| 254 | + } |
| 255 | +} |
| 256 | + |
| 257 | +export const seedOptions = { |
| 258 | + validateFn: validate, |
| 259 | + ttlSeconds: CACHE_TTL_SECONDS, |
| 260 | + sourceVersion: 'idmc-idu-hdx-v1', |
| 261 | + declareRecords, |
| 262 | + contentMeta, |
| 263 | + schemaVersion: 1, |
| 264 | + maxStaleMin: 2880, // daily health-bundle tick; 48h = two missed ticks |
| 265 | + // IDMC publishes new figures most weekdays. |
| 266 | + maxContentAgeMin: 14 * DAY_MIN, |
| 267 | + emptyDataIsFailure: true, |
| 268 | + afterPublish: markActivated, |
| 269 | +}; |
| 270 | + |
| 271 | +if (process.argv[1]?.endsWith('seed-idmc-idu-displacement.mjs')) { |
| 272 | + runSeed('displacement', 'idmc-idu', IDU_CANONICAL_KEY, () => fetchIduDisplacement(), seedOptions).catch((err) => { |
| 273 | + const cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; |
| 274 | + console.error('FATAL:', (err.message || err) + cause); |
| 275 | + process.exit(1); |
| 276 | + }); |
| 277 | +} |
0 commit comments