函数实现
// 生成的箭头函数代码
const fetchAllPages = (startUrl, { retry = 2, backoff = 300, timeout = 8000, signal } = {}) => {
if (typeof startUrl !== 'string' || !startUrl.trim()) {
throw new TypeError('fetchAllPages: startUrl must be a non-empty string');
}
// helpers
const now = typeof performance !== 'undefined' && performance && typeof performance.now === 'function'
? () => performance.now()
: () => Date.now();
const createTimeoutError = (ms) => {
try {
return new DOMException(`Request timed out after ${ms}ms`, 'TimeoutError');
} catch {
const e = new Error(`Request timed out after ${ms}ms`);
e.name = 'TimeoutError';
return e;
}
};
const sleep = (ms, abortSignal) => new Promise((resolve, reject) => {
if (ms <= 0) return resolve();
let tid;
const cleanup = () => {
if (tid) clearTimeout(tid);
if (abortSignal) abortSignal.removeEventListener('abort', onAbort);
};
const onAbort = () => {
cleanup();
reject(abortSignal.reason || new DOMException('Aborted', 'AbortError'));
};
if (abortSignal) {
if (abortSignal.aborted) return onAbort();
abortSignal.addEventListener('abort', onAbort, { once: true });
}
tid = setTimeout(() => {
cleanup();
resolve();
}, ms);
});
const addJitter = (ms) => Math.floor(Math.random() * ms); // full jitter [0..ms)
const linkAbort = (srcSignal, dstController) => {
if (!srcSignal) return () => {};
const forward = () => dstController.abort(srcSignal.reason || new DOMException('Aborted', 'AbortError'));
if (srcSignal.aborted) {
forward();
return () => {};
}
srcSignal.addEventListener('abort', forward, { once: true });
return () => srcSignal.removeEventListener('abort', forward);
};
// outer controller governs the whole task
const outerController = new AbortController();
const unlinkExternal = linkAbort(signal, outerController);
let settled = false;
const cancel = (reason = new DOMException('Aborted', 'AbortError')) => {
if (!outerController.signal.aborted) outerController.abort(reason);
};
const fetchPage = async (url) => {
let attempt = 0;
// retry loop
// retry applies to 429, timeout, and network errors only
// other HTTP errors fail fast
// backoff: exponential with full jitter
while (true) {
attempt += 1;
const attemptController = new AbortController();
const unlinkOuter = linkAbort(outerController.signal, attemptController);
let timer;
const timeoutPromise = new Promise((_, reject) => {
timer = setTimeout(() => {
const err = createTimeoutError(timeout);
// abort the fetch to avoid dangling work
attemptController.abort(err);
reject(err);
}, timeout);
});
try {
const res = await Promise.race([
fetch(url, { signal: attemptController.signal, headers: { accept: 'application/json' } }),
timeoutPromise
]);
// If the race timed out, the above await would have thrown; reaching here means fetch resolved
if (res.status === 429) {
const err = new Error('Too Many Requests');
err.code = 'HTTP_429';
throw err;
}
if (!res.ok) {
const err = new Error(`HTTP ${res.status} ${res.statusText || ''}`.trim());
err.code = 'HTTP_ERROR';
err.status = res.status;
throw err;
}
// Robust JSON parse; will throw on invalid JSON
const data = await res.json();
return data;
} catch (err) {
// If the whole task was cancelled, propagate immediately
if (outerController.signal.aborted) {
throw outerController.signal.reason || err;
}
// Decide retryability
const isTimeout = err && err.name === 'TimeoutError';
const isNetwork = err && err.name === 'TypeError'; // fetch network error in browsers
const isInnerAbortTimeout = err && err.name === 'AbortError' && !outerController.signal.aborted; // our per-attempt abort due to timeout
const is429 = err && err.code === 'HTTP_429';
const retryable = is429 || isTimeout || isNetwork || isInnerAbortTimeout;
if (!retryable || attempt > retry) {
throw err;
}
const delay = addJitter(backoff * (2 ** (attempt - 1)));
await sleep(delay, outerController.signal);
// retry
} finally {
if (timer) clearTimeout(timer);
unlinkOuter();
}
}
};
const t0 = now();
const promise = (async () => {
try {
const byId = new Map(); // id -> record
const idless = []; // records without numeric/string id
const seen = new Set(); // detect accidental loops
let url = startUrl;
while (url) {
if (outerController.signal.aborted) {
throw outerController.signal.reason || new DOMException('Aborted', 'AbortError');
}
if (seen.has(url)) {
throw new Error('fetchAllPages: detected circular "next" references');
}
seen.add(url);
const { records = [], next = null } = await fetchPage(url);
if (Array.isArray(records)) {
for (const rec of records) {
const id = rec != null && (typeof rec.id === 'string' || typeof rec.id === 'number') ? rec.id : undefined;
if (id === undefined) {
idless.push(rec);
} else if (!byId.has(id)) {
byId.set(id, rec);
}
}
}
url = next || null;
}
const resultRecords = [...byId.values(), ...idless];
const durationMs = Math.round(now() - t0);
settled = true;
return { records: resultRecords, durationMs };
} finally {
unlinkExternal();
}
})();
return Object.assign(promise, { cancel });
};
代码说明
-
参数说明：
- startUrl: string，分页抓取的起始地址。
- options:
- retry: number，重试次数（对每一页的请求，默认 2 次，指失败后再试的次数）。
- backoff: number，首次重试等待基数（毫秒，默认 300），使用指数退避和全随机抖动：backoff * 2^(attempt-1)，并在 [0..delay) 之间随机。
- timeout: number，请求超时时间（毫秒，默认 8000），超时将中断该次 fetch 并按重试策略重试。
- signal: AbortSignal，可选，外部传入的取消信号（如路由切换时触发）。
-
返回值说明：
- 返回一个可等待的 Promise，并带有 cancel 方法。
- await 返回值：{ records: Array, durationMs: number }
- records: 合并并去重后的记录数组，以 record.id（string 或 number）唯一；没有 id 的记录会保留，但不会参与去重。
- durationMs: 整体抓取耗时（毫秒）。
- 取消：通过返回值的 cancel() 主动中断整个过程；或通过传入的 signal 中断。取消会使 Promise 以 AbortError 拒绝。
-
使用示例：
- 基本使用
// 初始化数据看板
const task = fetchAllPages('/api/metrics?cursor=start', {
retry: 3,
backoff: 400,
timeout: 7000
});
try {
const { records, durationMs } = await task;
console.log('records:', records.length, 'duration:', durationMs, 'ms');
} catch (e) {
if (e.name === 'AbortError') {
console.warn('fetchAllPages cancelled');
} else {
console.error('fetchAllPages failed:', e);
}
}
- 路由切换时中断
const task = fetchAllPages('/api/metrics?cursor=start');
// 路由切换时
router.beforeEach(() => {
task.cancel(); // 中断所有未完成请求
});
```
// 某处触发
controller.abort(); // 将使 task 以 AbortError 失败
```
- 注意事项：
- 响应格式要求：每页结果为 JSON，形如 { records: Array, next: string|null }。next 为空或假值表示结束。
- 去重规则：仅对具有 string/number 类型 id 的记录去重；无 id 的记录全部保留。
- 重试范围：仅对 429、网络错误（fetch 在浏览器中通常为 TypeError）以及超时进行重试；其他 HTTP 非 2xx（除 429）直接失败。
- 超时实现：使用 Promise.race 控制整体超时，同时通过 AbortController 主动中止 fetch，避免资源泄露。
- 取消语义：调用 cancel() 或外部 signal 触发时，将立即中止当前与后续请求，Promise 以 AbortError 拒绝，不返回部分结果。
- 循环保护：若 next 出现循环引用，将抛出错误以避免无限抓取。
- 环境要求：浏览器原生 fetch 与 AbortController。若在不支持 DOMException 的环境，错误对象将退化为 name 为 'TimeoutError' 的 Error。
