同步记录
This commit is contained in:
1 parent
818c980dec
commit
d647453085
10 files changed
+439
-65
No files matched your search
@@ -142,7 +142,7 @@ function formatTime(value: string) {
|
|||||||
{{ t('storage.backup.colProgress') }}
|
{{ t('storage.backup.colProgress') }}
|
||||||
</th>
|
</th>
|
||||||
<th class="px-4 py-3 font-medium sm:px-5">
|
<th class="px-4 py-3 font-medium sm:px-5">
|
||||||
{{ t('storage.backup.colCreated') }}
|
{{ t('storage.backup.colUpdated') }}
|
||||||
</th>
|
</th>
|
||||||
<th class="px-4 py-3 font-medium sm:px-5">
|
<th class="px-4 py-3 font-medium sm:px-5">
|
||||||
{{ t('storage.backup.colActions') }}
|
{{ t('storage.backup.colActions') }}
|
||||||
@@ -203,7 +203,7 @@ function formatTime(value: string) {
|
|||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
<td class="px-4 py-3.5 whitespace-nowrap text-xs text-muted sm:px-5">
|
<td class="px-4 py-3.5 whitespace-nowrap text-xs text-muted sm:px-5">
|
||||||
{{ formatTime(job.createdAt) }}
|
{{ formatTime(job.updatedAt || job.finishedAt || job.createdAt) }}
|
||||||
</td>
|
</td>
|
||||||
<td class="px-4 py-3.5 sm:px-5">
|
<td class="px-4 py-3.5 sm:px-5">
|
||||||
<div class="flex flex-wrap gap-1.5">
|
<div class="flex flex-wrap gap-1.5">
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ export interface BackupJobItem {
|
|||||||
downloadable: boolean
|
downloadable: boolean
|
||||||
error: string | null
|
error: string | null
|
||||||
createdAt: string
|
createdAt: string
|
||||||
|
updatedAt: string
|
||||||
finishedAt: string | null
|
finishedAt: string | null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -745,6 +745,7 @@
|
|||||||
"progressUnitObjects": "storage objects",
|
"progressUnitObjects": "storage objects",
|
||||||
"colMessage": "Message",
|
"colMessage": "Message",
|
||||||
"colCreated": "Created",
|
"colCreated": "Created",
|
||||||
|
"colUpdated": "Updated",
|
||||||
"colActions": "Actions",
|
"colActions": "Actions",
|
||||||
"jobTypeExport": "Site export",
|
"jobTypeExport": "Site export",
|
||||||
"jobTypeRestore": "Site restore",
|
"jobTypeRestore": "Site restore",
|
||||||
|
|||||||
@@ -745,6 +745,7 @@
|
|||||||
"progressUnitObjects": "存储对象",
|
"progressUnitObjects": "存储对象",
|
||||||
"colMessage": "说明",
|
"colMessage": "说明",
|
||||||
"colCreated": "创建时间",
|
"colCreated": "创建时间",
|
||||||
|
"colUpdated": "更新时间",
|
||||||
"colActions": "操作",
|
"colActions": "操作",
|
||||||
"jobTypeExport": "站点导出",
|
"jobTypeExport": "站点导出",
|
||||||
"jobTypeRestore": "站点恢复",
|
"jobTypeRestore": "站点恢复",
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ export function serializeBackupJob(job: BackupJob) {
|
|||||||
),
|
),
|
||||||
error: job.error,
|
error: job.error,
|
||||||
createdAt: job.createdAt,
|
createdAt: job.createdAt,
|
||||||
|
updatedAt: job.updatedAt,
|
||||||
finishedAt: job.finishedAt
|
finishedAt: job.finishedAt
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+32
-11
@@ -17,6 +17,7 @@ export interface BackupJob {
|
|||||||
outputPath: string | null
|
outputPath: string | null
|
||||||
error: string | null
|
error: string | null
|
||||||
createdAt: string
|
createdAt: string
|
||||||
|
updatedAt: string
|
||||||
finishedAt: string | null
|
finishedAt: string | null
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -32,6 +33,7 @@ interface BackupJobRow {
|
|||||||
output_path: string | null
|
output_path: string | null
|
||||||
error: string | null
|
error: string | null
|
||||||
created_at: string
|
created_at: string
|
||||||
|
updated_at: string
|
||||||
finished_at: string | null
|
finished_at: string | null
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,6 +57,7 @@ function rowToJob(row: BackupJobRow): BackupJob {
|
|||||||
outputPath: row.output_path,
|
outputPath: row.output_path,
|
||||||
error: row.error,
|
error: row.error,
|
||||||
createdAt: row.created_at,
|
createdAt: row.created_at,
|
||||||
|
updatedAt: row.updated_at,
|
||||||
finishedAt: row.finished_at
|
finishedAt: row.finished_at
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -69,7 +72,7 @@ export function getRunningBackupJob(userId?: number): BackupJob | null {
|
|||||||
ensureBackupSchema()
|
ensureBackupSchema()
|
||||||
const row = getDb().prepare(`
|
const row = getDb().prepare(`
|
||||||
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
||||||
output_path, error, created_at, finished_at
|
output_path, error, created_at, updated_at, finished_at
|
||||||
FROM backup_jobs
|
FROM backup_jobs
|
||||||
WHERE status = 'running'${userId === undefined ? '' : ' AND user_id = ?'}
|
WHERE status = 'running'${userId === undefined ? '' : ' AND user_id = ?'}
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
@@ -81,7 +84,7 @@ export function getActiveBackupJob(userId?: number): BackupJob | null {
|
|||||||
ensureBackupSchema()
|
ensureBackupSchema()
|
||||||
const row = getDb().prepare(`
|
const row = getDb().prepare(`
|
||||||
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
||||||
output_path, error, created_at, finished_at
|
output_path, error, created_at, updated_at, finished_at
|
||||||
FROM backup_jobs
|
FROM backup_jobs
|
||||||
WHERE status IN ('pending', 'running')${userId === undefined ? '' : ' AND user_id = ?'}
|
WHERE status IN ('pending', 'running')${userId === undefined ? '' : ' AND user_id = ?'}
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
@@ -105,8 +108,8 @@ export function createBackupJob(input: {
|
|||||||
|
|
||||||
getDb().prepare(`
|
getDb().prepare(`
|
||||||
INSERT INTO backup_jobs
|
INSERT INTO backup_jobs
|
||||||
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
|
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, updated_at, finished_at)
|
||||||
VALUES (?, ?, ?, 'pending', 0, ?, ?, ?, NULL, NULL, ?, NULL)
|
VALUES (?, ?, ?, 'pending', 0, ?, ?, ?, NULL, NULL, ?, ?, NULL)
|
||||||
`).run(
|
`).run(
|
||||||
id,
|
id,
|
||||||
userId,
|
userId,
|
||||||
@@ -114,6 +117,7 @@ export function createBackupJob(input: {
|
|||||||
input.total ?? 0,
|
input.total ?? 0,
|
||||||
input.message ?? '',
|
input.message ?? '',
|
||||||
payloadJson,
|
payloadJson,
|
||||||
|
now,
|
||||||
now
|
now
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -124,7 +128,7 @@ export function getBackupJob(id: string, userId?: number): BackupJob | null {
|
|||||||
ensureBackupSchema()
|
ensureBackupSchema()
|
||||||
const row = getDb().prepare(`
|
const row = getDb().prepare(`
|
||||||
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
||||||
output_path, error, created_at, finished_at
|
output_path, error, created_at, updated_at, finished_at
|
||||||
FROM backup_jobs
|
FROM backup_jobs
|
||||||
WHERE id = ?${userId === undefined ? '' : ' AND user_id = ?'}
|
WHERE id = ?${userId === undefined ? '' : ' AND user_id = ?'}
|
||||||
`).get(...(userId === undefined ? [id] : [id, userId])) as BackupJobRow | undefined
|
`).get(...(userId === undefined ? [id] : [id, userId])) as BackupJobRow | undefined
|
||||||
@@ -156,7 +160,7 @@ export function listBackupJobsPage(
|
|||||||
const total = countBackupJobs(userId)
|
const total = countBackupJobs(userId)
|
||||||
const rows = getDb().prepare(`
|
const rows = getDb().prepare(`
|
||||||
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
||||||
output_path, error, created_at, finished_at
|
output_path, error, created_at, updated_at, finished_at
|
||||||
FROM backup_jobs
|
FROM backup_jobs
|
||||||
${userId === undefined ? '' : 'WHERE user_id = ?'}
|
${userId === undefined ? '' : 'WHERE user_id = ?'}
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
@@ -169,7 +173,7 @@ export function listAllBackupJobsForUser(userId: number): BackupJob[] {
|
|||||||
ensureBackupSchema()
|
ensureBackupSchema()
|
||||||
const rows = getDb().prepare(`
|
const rows = getDb().prepare(`
|
||||||
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
SELECT id, user_id, type, status, progress, total, message, payload_json,
|
||||||
output_path, error, created_at, finished_at
|
output_path, error, created_at, updated_at, finished_at
|
||||||
FROM backup_jobs
|
FROM backup_jobs
|
||||||
WHERE user_id = ?
|
WHERE user_id = ?
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
@@ -197,8 +201,8 @@ export function restoreBackupJobsAfterReplace(preserved: BackupJob[]): void {
|
|||||||
db.prepare('DELETE FROM backup_jobs').run()
|
db.prepare('DELETE FROM backup_jobs').run()
|
||||||
const insert = db.prepare(`
|
const insert = db.prepare(`
|
||||||
INSERT INTO backup_jobs
|
INSERT INTO backup_jobs
|
||||||
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
|
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, updated_at, finished_at)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
`)
|
`)
|
||||||
for (const job of preserved) {
|
for (const job of preserved) {
|
||||||
insert.run(
|
insert.run(
|
||||||
@@ -213,6 +217,7 @@ export function restoreBackupJobsAfterReplace(preserved: BackupJob[]): void {
|
|||||||
job.outputPath,
|
job.outputPath,
|
||||||
job.error,
|
job.error,
|
||||||
job.createdAt,
|
job.createdAt,
|
||||||
|
job.updatedAt,
|
||||||
job.finishedAt
|
job.finishedAt
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -232,9 +237,10 @@ export function recoverInterruptedBackupJobs(): number {
|
|||||||
SET status = 'failed',
|
SET status = 'failed',
|
||||||
error = '任务中断(服务重启或异常退出)',
|
error = '任务中断(服务重启或异常退出)',
|
||||||
message = '任务已中断,请重试',
|
message = '任务已中断,请重试',
|
||||||
|
updated_at = ?,
|
||||||
finished_at = ?
|
finished_at = ?
|
||||||
WHERE status = 'running'
|
WHERE status = 'running'
|
||||||
`).run(finishedAt)
|
`).run(finishedAt, finishedAt)
|
||||||
return Number(result.changes ?? 0)
|
return Number(result.changes ?? 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -263,13 +269,14 @@ export function updateBackupJob(
|
|||||||
payload: patch.payload ?? current.payload,
|
payload: patch.payload ?? current.payload,
|
||||||
outputPath: patch.outputPath !== undefined ? patch.outputPath : current.outputPath,
|
outputPath: patch.outputPath !== undefined ? patch.outputPath : current.outputPath,
|
||||||
error: patch.error !== undefined ? patch.error : current.error,
|
error: patch.error !== undefined ? patch.error : current.error,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
finishedAt: patch.finishedAt !== undefined ? patch.finishedAt : current.finishedAt
|
finishedAt: patch.finishedAt !== undefined ? patch.finishedAt : current.finishedAt
|
||||||
}
|
}
|
||||||
|
|
||||||
getDb().prepare(`
|
getDb().prepare(`
|
||||||
UPDATE backup_jobs
|
UPDATE backup_jobs
|
||||||
SET status = ?, progress = ?, total = ?, message = ?, payload_json = ?,
|
SET status = ?, progress = ?, total = ?, message = ?, payload_json = ?,
|
||||||
output_path = ?, error = ?, finished_at = ?
|
output_path = ?, error = ?, updated_at = ?, finished_at = ?
|
||||||
WHERE id = ?
|
WHERE id = ?
|
||||||
`).run(
|
`).run(
|
||||||
next.status,
|
next.status,
|
||||||
@@ -279,11 +286,25 @@ export function updateBackupJob(
|
|||||||
JSON.stringify(next.payload),
|
JSON.stringify(next.payload),
|
||||||
next.outputPath,
|
next.outputPath,
|
||||||
next.error,
|
next.error,
|
||||||
|
next.updatedAt,
|
||||||
next.finishedAt,
|
next.finishedAt,
|
||||||
id
|
id
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Update job data without changing its last activity timestamp. */
|
||||||
|
export function updateBackupJobPayload(
|
||||||
|
id: string,
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
): void {
|
||||||
|
ensureBackupSchema()
|
||||||
|
getDb().prepare(`
|
||||||
|
UPDATE backup_jobs
|
||||||
|
SET payload_json = ?
|
||||||
|
WHERE id = ?
|
||||||
|
`).run(JSON.stringify(payload), id)
|
||||||
|
}
|
||||||
|
|
||||||
export function markJobRunning(id: string, message?: string): void {
|
export function markJobRunning(id: string, message?: string): void {
|
||||||
updateBackupJob(id, {
|
updateBackupJob(id, {
|
||||||
status: 'running',
|
status: 'running',
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
markJobFailed,
|
markJobFailed,
|
||||||
markJobRunning,
|
markJobRunning,
|
||||||
recoverInterruptedBackupJobs,
|
recoverInterruptedBackupJobs,
|
||||||
|
updateBackupJobPayload,
|
||||||
type BackupJob
|
type BackupJob
|
||||||
} from './jobs'
|
} from './jobs'
|
||||||
import {
|
import {
|
||||||
@@ -149,22 +150,41 @@ export function resumeSiteSyncJob(
|
|||||||
userId?: number | null
|
userId?: number | null
|
||||||
): BackupJob {
|
): BackupJob {
|
||||||
claimJobSlot()
|
claimJobSlot()
|
||||||
const job = getBackupJob(jobId, userId ?? undefined)
|
const previousJob = getBackupJob(jobId, userId ?? undefined)
|
||||||
if (
|
if (
|
||||||
!job
|
!previousJob
|
||||||
|| job.type !== 'site_sync'
|
|| previousJob.type !== 'site_sync'
|
||||||
|| job.status !== 'failed'
|
|| previousJob.status !== 'failed'
|
||||||
|| job.payload.resumable !== true
|
|| previousJob.payload.resumable !== true
|
||||||
) {
|
) {
|
||||||
throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
||||||
}
|
}
|
||||||
const targetBaseUrl = typeof job.payload.targetBaseUrl === 'string'
|
const targetBaseUrl = typeof previousJob.payload.targetBaseUrl === 'string'
|
||||||
? job.payload.targetBaseUrl
|
? previousJob.payload.targetBaseUrl
|
||||||
: ''
|
: ''
|
||||||
const conflictMode = job.payload.conflictMode === 'skip' ? 'skip' : 'overwrite'
|
const conflictMode = previousJob.payload.conflictMode === 'skip'
|
||||||
const direction = job.payload.direction === 'pull' ? 'pull' : 'push'
|
? 'skip'
|
||||||
|
: 'overwrite'
|
||||||
|
const direction = previousJob.payload.direction === 'pull' ? 'pull' : 'push'
|
||||||
if (!targetBaseUrl) throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
if (!targetBaseUrl) throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
||||||
|
|
||||||
|
const job = createBackupJob({
|
||||||
|
type: 'site_sync',
|
||||||
|
userId: previousJob.userId,
|
||||||
|
total: previousJob.total,
|
||||||
|
message: `继续任务 ${previousJob.id}…`,
|
||||||
|
payload: {
|
||||||
|
...previousJob.payload,
|
||||||
|
resumedFromJobId: previousJob.id,
|
||||||
|
resumedAt: new Date().toISOString(),
|
||||||
|
cancelled: false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
updateBackupJobPayload(previousJob.id, {
|
||||||
|
...previousJob.payload,
|
||||||
|
resumable: false,
|
||||||
|
supersededByJobId: job.id
|
||||||
|
})
|
||||||
registerSiteSyncToken(job.id, token)
|
registerSiteSyncToken(job.id, token)
|
||||||
scheduleBackupJob(job, () => runSiteSyncJob(job.id, {
|
scheduleBackupJob(job, () => runSiteSyncJob(job.id, {
|
||||||
targetBaseUrl,
|
targetBaseUrl,
|
||||||
|
|||||||
@@ -24,11 +24,12 @@ function migrateBackupJobTypes(): void {
|
|||||||
output_path TEXT,
|
output_path TEXT,
|
||||||
error TEXT,
|
error TEXT,
|
||||||
created_at TEXT NOT NULL,
|
created_at TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL,
|
||||||
finished_at TEXT
|
finished_at TEXT
|
||||||
);
|
);
|
||||||
INSERT INTO backup_jobs_new
|
INSERT INTO backup_jobs_new
|
||||||
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
|
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, updated_at, finished_at)
|
||||||
SELECT id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at
|
SELECT id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, COALESCE(updated_at, created_at), finished_at
|
||||||
FROM backup_jobs;
|
FROM backup_jobs;
|
||||||
DROP TABLE backup_jobs;
|
DROP TABLE backup_jobs;
|
||||||
ALTER TABLE backup_jobs_new RENAME TO backup_jobs;
|
ALTER TABLE backup_jobs_new RENAME TO backup_jobs;
|
||||||
@@ -53,6 +54,7 @@ export function ensureBackupSchema(): void {
|
|||||||
output_path TEXT,
|
output_path TEXT,
|
||||||
error TEXT,
|
error TEXT,
|
||||||
created_at TEXT NOT NULL,
|
created_at TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL,
|
||||||
finished_at TEXT
|
finished_at TEXT
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_backup_jobs_created_at
|
CREATE INDEX IF NOT EXISTS idx_backup_jobs_created_at
|
||||||
@@ -65,6 +67,9 @@ export function ensureBackupSchema(): void {
|
|||||||
if (!columns.some(column => column.name === 'user_id')) {
|
if (!columns.some(column => column.name === 'user_id')) {
|
||||||
getDb().exec('ALTER TABLE backup_jobs ADD COLUMN user_id INTEGER')
|
getDb().exec('ALTER TABLE backup_jobs ADD COLUMN user_id INTEGER')
|
||||||
}
|
}
|
||||||
|
if (!columns.some(column => column.name === 'updated_at')) {
|
||||||
|
getDb().exec('ALTER TABLE backup_jobs ADD COLUMN updated_at TEXT')
|
||||||
|
}
|
||||||
getDb().exec(`
|
getDb().exec(`
|
||||||
CREATE INDEX IF NOT EXISTS idx_backup_jobs_user_created
|
CREATE INDEX IF NOT EXISTS idx_backup_jobs_user_created
|
||||||
ON backup_jobs(user_id, created_at DESC);
|
ON backup_jobs(user_id, created_at DESC);
|
||||||
@@ -74,6 +79,13 @@ export function ensureBackupSchema(): void {
|
|||||||
)
|
)
|
||||||
WHERE user_id IS NULL
|
WHERE user_id IS NULL
|
||||||
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
|
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
|
||||||
|
UPDATE backup_jobs
|
||||||
|
SET updated_at = COALESCE(finished_at, created_at)
|
||||||
|
WHERE updated_at IS NULL;
|
||||||
|
UPDATE backup_jobs
|
||||||
|
SET updated_at = finished_at
|
||||||
|
WHERE finished_at IS NOT NULL
|
||||||
|
AND updated_at = created_at;
|
||||||
`)
|
`)
|
||||||
migrateBackupJobTypes()
|
migrateBackupJobTypes()
|
||||||
}
|
}
|
||||||
@@ -62,7 +62,11 @@ describe('fetchSiteSyncJson', () => {
|
|||||||
it('stops retrying immediately when the job is cancelled', async () => {
|
it('stops retrying immediately when the job is cancelled', async () => {
|
||||||
const jobId = 'cancel-test'
|
const jobId = 'cancel-test'
|
||||||
beginJobCancellationScope(jobId)
|
beginJobCancellationScope(jobId)
|
||||||
const operation = vi.fn().mockRejectedValue(new Error('connection reset'))
|
const operation = vi.fn().mockImplementation(() => (
|
||||||
|
new Promise((_, reject) => {
|
||||||
|
setTimeout(() => reject(new Error('connection reset')), 1000)
|
||||||
|
})
|
||||||
|
))
|
||||||
const result = retrySiteSyncOperation(operation, getJobAbortSignal(jobId))
|
const result = retrySiteSyncOperation(operation, getJobAbortSignal(jobId))
|
||||||
|
|
||||||
setTimeout(() => requestJobCancellation(jobId), 25)
|
setTimeout(() => requestJobCancellation(jobId), 25)
|
||||||
@@ -71,4 +75,40 @@ describe('fetchSiteSyncJson', () => {
|
|||||||
expect(operation).toHaveBeenCalledTimes(1)
|
expect(operation).toHaveBeenCalledTimes(1)
|
||||||
endJobCancellationScope(jobId)
|
endJobCancellationScope(jobId)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('reports every retry attempt and backoff to the job status', async () => {
|
||||||
|
const fetchMock = vi.fn()
|
||||||
|
.mockRejectedValueOnce(new DOMException('timed out', 'TimeoutError'))
|
||||||
|
.mockResolvedValueOnce(new Response('{"ok":true}', {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
}))
|
||||||
|
vi.stubGlobal('fetch', fetchMock)
|
||||||
|
const statuses: Array<{
|
||||||
|
attempt: number
|
||||||
|
phase: string
|
||||||
|
delayMs?: number
|
||||||
|
}> = []
|
||||||
|
|
||||||
|
await expect(fetchSiteSyncJson(
|
||||||
|
'http://target/api',
|
||||||
|
'token',
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
status => statuses.push(status)
|
||||||
|
)).resolves.toEqual({ ok: true })
|
||||||
|
|
||||||
|
expect(
|
||||||
|
statuses
|
||||||
|
.filter(status => status.phase !== 'countdown')
|
||||||
|
.map(status => `${status.phase}:${status.attempt}`)
|
||||||
|
).toEqual([
|
||||||
|
'attempt:1',
|
||||||
|
'retry:1',
|
||||||
|
'attempt:2'
|
||||||
|
])
|
||||||
|
expect(
|
||||||
|
statuses.find(status => status.phase === 'retry')?.delayMs
|
||||||
|
).toBe(0)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
@@ -1,4 +1,8 @@
|
|||||||
import { basename } from 'node:path'
|
import { basename } from 'node:path'
|
||||||
|
import {
|
||||||
|
Agent,
|
||||||
|
fetch as undiciFetch
|
||||||
|
} from 'undici'
|
||||||
import type { Readable } from 'node:stream'
|
import type { Readable } from 'node:stream'
|
||||||
import { contentTypeFromKey } from '../content-type'
|
import { contentTypeFromKey } from '../content-type'
|
||||||
import {
|
import {
|
||||||
@@ -26,7 +30,6 @@ import {
|
|||||||
import {
|
import {
|
||||||
createJobCancelledError,
|
createJobCancelledError,
|
||||||
getJobAbortSignal,
|
getJobAbortSignal,
|
||||||
getJobOperationSignal,
|
|
||||||
isJobCancellationRequested,
|
isJobCancellationRequested,
|
||||||
isJobCancelledError,
|
isJobCancelledError,
|
||||||
throwIfJobCancellationRequested
|
throwIfJobCancellationRequested
|
||||||
@@ -62,8 +65,32 @@ export function registerSiteSyncToken(jobId: string, token: string): void {
|
|||||||
tokens.set(jobId, token)
|
tokens.set(jobId, token)
|
||||||
}
|
}
|
||||||
|
|
||||||
const SITE_SYNC_JSON_TIMEOUT_MS = 2 * 60 * 1000
|
const SITE_SYNC_CONNECT_TIMEOUT_MS = 3 * 1000
|
||||||
const SITE_SYNC_OBJECT_TIMEOUT_MS = 15 * 60 * 1000
|
const SITE_SYNC_RESPONSE_IDLE_TIMEOUT_MS = 3 * 1000
|
||||||
|
const SITE_SYNC_UPLOAD_HEADERS_TIMEOUT_MS = 10 * 60 * 1000
|
||||||
|
|
||||||
|
interface SiteSyncAgents {
|
||||||
|
upload: Agent
|
||||||
|
object: Agent
|
||||||
|
}
|
||||||
|
|
||||||
|
const agents = (globalThis as typeof globalThis & {
|
||||||
|
__pichostSiteSyncAgents?: SiteSyncAgents
|
||||||
|
}).__pichostSiteSyncAgents ?? {
|
||||||
|
upload: new Agent({
|
||||||
|
connect: { timeout: SITE_SYNC_CONNECT_TIMEOUT_MS },
|
||||||
|
headersTimeout: SITE_SYNC_UPLOAD_HEADERS_TIMEOUT_MS,
|
||||||
|
bodyTimeout: SITE_SYNC_RESPONSE_IDLE_TIMEOUT_MS
|
||||||
|
}),
|
||||||
|
object: new Agent({
|
||||||
|
connect: { timeout: SITE_SYNC_CONNECT_TIMEOUT_MS },
|
||||||
|
headersTimeout: SITE_SYNC_RESPONSE_IDLE_TIMEOUT_MS,
|
||||||
|
bodyTimeout: SITE_SYNC_RESPONSE_IDLE_TIMEOUT_MS
|
||||||
|
})
|
||||||
|
}
|
||||||
|
;(globalThis as typeof globalThis & {
|
||||||
|
__pichostSiteSyncAgents?: SiteSyncAgents
|
||||||
|
}).__pichostSiteSyncAgents = agents
|
||||||
|
|
||||||
async function streamToBuffer(
|
async function streamToBuffer(
|
||||||
stream: Readable,
|
stream: Readable,
|
||||||
@@ -84,7 +111,10 @@ function targetUrl(baseUrl: string, path: string): string {
|
|||||||
return `${baseUrl.replace(/\/+$/, '')}${path}`
|
return `${baseUrl.replace(/\/+$/, '')}${path}`
|
||||||
}
|
}
|
||||||
|
|
||||||
async function responseError(response: Response): Promise<Error> {
|
async function responseError(response: {
|
||||||
|
status: number
|
||||||
|
json: () => Promise<unknown>
|
||||||
|
}): Promise<Error> {
|
||||||
const payload = await response.json().catch(() => null) as {
|
const payload = await response.json().catch(() => null) as {
|
||||||
message?: string
|
message?: string
|
||||||
statusMessage?: string
|
statusMessage?: string
|
||||||
@@ -92,11 +122,60 @@ async function responseError(response: Response): Promise<Error> {
|
|||||||
return new Error(payload?.message || payload?.statusMessage || `目标站点返回 ${response.status}`)
|
return new Error(payload?.message || payload?.statusMessage || `目标站点返回 ${response.status}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseResponseErrorText(
|
||||||
|
text: string,
|
||||||
|
status: number
|
||||||
|
): Error {
|
||||||
|
try {
|
||||||
|
const payload = JSON.parse(text) as {
|
||||||
|
message?: string
|
||||||
|
statusMessage?: string
|
||||||
|
}
|
||||||
|
return new Error(
|
||||||
|
payload.message
|
||||||
|
|| payload.statusMessage
|
||||||
|
|| `目标站点返回 ${status}`
|
||||||
|
)
|
||||||
|
} catch {
|
||||||
|
return new Error(`目标站点返回 ${status}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const SITE_SYNC_MAX_ATTEMPTS = 3
|
const SITE_SYNC_MAX_ATTEMPTS = 3
|
||||||
|
|
||||||
|
export interface SiteSyncRetryStatus {
|
||||||
|
attempt: number
|
||||||
|
maxAttempts: number
|
||||||
|
phase: 'attempt' | 'countdown' | 'retry'
|
||||||
|
delayMs?: number
|
||||||
|
remainingMs?: number
|
||||||
|
error?: Error
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatRetryStatus(
|
||||||
|
step: string,
|
||||||
|
status: SiteSyncRetryStatus
|
||||||
|
): string {
|
||||||
|
if (status.phase === 'attempt') {
|
||||||
|
return `${step}(第 ${status.attempt}/${status.maxAttempts} 次)`
|
||||||
|
}
|
||||||
|
if (status.phase === 'countdown') {
|
||||||
|
const seconds = Math.max(
|
||||||
|
1,
|
||||||
|
Math.ceil((status.remainingMs ?? 0) / 1000)
|
||||||
|
)
|
||||||
|
return `${step}(第 ${status.attempt}/${status.maxAttempts} 次)· 连接倒计时 ${seconds}s`
|
||||||
|
}
|
||||||
|
if ((status.delayMs ?? 0) < 1000) {
|
||||||
|
return `${step}第 ${status.attempt}/${status.maxAttempts} 次无响应,即将重试:${status.error?.message ?? '连接失败'}`
|
||||||
|
}
|
||||||
|
const seconds = Math.max(1, Math.round((status.delayMs ?? 0) / 1000))
|
||||||
|
return `${step}第 ${status.attempt}/${status.maxAttempts} 次失败,${seconds} 秒后重试:${status.error?.message ?? '连接失败'}`
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeSiteSyncError(error: Error): Error {
|
function normalizeSiteSyncError(error: Error): Error {
|
||||||
if (error.name === 'TimeoutError') {
|
if (/timeout/i.test(error.name) || /timed out|idle for too long/i.test(error.message)) {
|
||||||
return new Error('目标站点连接超时或无响应')
|
return new Error('目标站点在 3 秒内未响应')
|
||||||
}
|
}
|
||||||
if (error instanceof TypeError && /fetch failed|terminated/i.test(error.message)) {
|
if (error instanceof TypeError && /fetch failed|terminated/i.test(error.message)) {
|
||||||
return new Error('目标站点无法连接')
|
return new Error('目标站点无法连接')
|
||||||
@@ -106,17 +185,57 @@ function normalizeSiteSyncError(error: Error): Error {
|
|||||||
|
|
||||||
export async function retrySiteSyncOperation<T>(
|
export async function retrySiteSyncOperation<T>(
|
||||||
operation: () => Promise<T>,
|
operation: () => Promise<T>,
|
||||||
signal?: AbortSignal
|
signal?: AbortSignal,
|
||||||
|
onStatus?: (status: SiteSyncRetryStatus) => void
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
if (signal?.aborted) throw createJobCancelledError()
|
if (signal?.aborted) throw createJobCancelledError()
|
||||||
let lastError: Error | null = null
|
let lastError: Error | null = null
|
||||||
for (let attempt = 1; attempt <= SITE_SYNC_MAX_ATTEMPTS; attempt++) {
|
for (let attempt = 1; attempt <= SITE_SYNC_MAX_ATTEMPTS; attempt++) {
|
||||||
|
let countdownTimer: ReturnType<typeof setInterval> | null = null
|
||||||
try {
|
try {
|
||||||
|
onStatus?.({
|
||||||
|
attempt,
|
||||||
|
maxAttempts: SITE_SYNC_MAX_ATTEMPTS,
|
||||||
|
phase: 'attempt'
|
||||||
|
})
|
||||||
|
if (onStatus) {
|
||||||
|
let remainingMs = SITE_SYNC_CONNECT_TIMEOUT_MS
|
||||||
|
onStatus({
|
||||||
|
attempt,
|
||||||
|
maxAttempts: SITE_SYNC_MAX_ATTEMPTS,
|
||||||
|
phase: 'countdown',
|
||||||
|
remainingMs
|
||||||
|
})
|
||||||
|
countdownTimer = setInterval(() => {
|
||||||
|
remainingMs -= 1000
|
||||||
|
if (remainingMs <= 0) {
|
||||||
|
if (countdownTimer) clearInterval(countdownTimer)
|
||||||
|
countdownTimer = null
|
||||||
|
return
|
||||||
|
}
|
||||||
|
onStatus({
|
||||||
|
attempt,
|
||||||
|
maxAttempts: SITE_SYNC_MAX_ATTEMPTS,
|
||||||
|
phase: 'countdown',
|
||||||
|
remainingMs
|
||||||
|
})
|
||||||
|
}, 1000)
|
||||||
|
}
|
||||||
return await operation()
|
return await operation()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (countdownTimer) clearInterval(countdownTimer)
|
||||||
|
countdownTimer = null
|
||||||
if (signal?.aborted) throw createJobCancelledError()
|
if (signal?.aborted) throw createJobCancelledError()
|
||||||
lastError = error instanceof Error ? error : new Error('两站点同步失败')
|
lastError = error instanceof Error ? error : new Error('两站点同步失败')
|
||||||
if (attempt < SITE_SYNC_MAX_ATTEMPTS) {
|
if (attempt < SITE_SYNC_MAX_ATTEMPTS) {
|
||||||
|
const delayMs = 0
|
||||||
|
onStatus?.({
|
||||||
|
attempt,
|
||||||
|
maxAttempts: SITE_SYNC_MAX_ATTEMPTS,
|
||||||
|
phase: 'retry',
|
||||||
|
delayMs,
|
||||||
|
error: normalizeSiteSyncError(lastError)
|
||||||
|
})
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
const onAbort = () => {
|
const onAbort = () => {
|
||||||
clearTimeout(timer)
|
clearTimeout(timer)
|
||||||
@@ -125,10 +244,12 @@ export async function retrySiteSyncOperation<T>(
|
|||||||
const timer = setTimeout(() => {
|
const timer = setTimeout(() => {
|
||||||
signal?.removeEventListener('abort', onAbort)
|
signal?.removeEventListener('abort', onAbort)
|
||||||
resolve()
|
resolve()
|
||||||
}, attempt * 500)
|
}, delayMs)
|
||||||
signal?.addEventListener('abort', onAbort, { once: true })
|
signal?.addEventListener('abort', onAbort, { once: true })
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
if (countdownTimer) clearInterval(countdownTimer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw lastError
|
throw lastError
|
||||||
@@ -141,34 +262,124 @@ async function fetchSiteSyncJsonOnce<T>(
|
|||||||
token: string,
|
token: string,
|
||||||
init?: RequestInit
|
init?: RequestInit
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
|
const connectionController = new AbortController()
|
||||||
|
const onAbort = () => connectionController.abort(createJobCancelledError())
|
||||||
|
init?.signal?.addEventListener('abort', onAbort, { once: true })
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
connectionController.abort(new DOMException(
|
||||||
|
'Target connection or TLS handshake timed out',
|
||||||
|
'TimeoutError'
|
||||||
|
))
|
||||||
|
}, SITE_SYNC_CONNECT_TIMEOUT_MS)
|
||||||
|
|
||||||
|
try {
|
||||||
const response = await fetch(url, {
|
const response = await fetch(url, {
|
||||||
...init,
|
...init,
|
||||||
headers: {
|
headers: {
|
||||||
'Auth-Token': token,
|
'Auth-Token': token,
|
||||||
...(init?.headers ?? {})
|
...(init?.headers ?? {})
|
||||||
}
|
},
|
||||||
|
signal: init?.signal
|
||||||
|
? AbortSignal.any([init.signal, connectionController.signal])
|
||||||
|
: connectionController.signal
|
||||||
|
})
|
||||||
|
clearTimeout(timeout)
|
||||||
|
|
||||||
|
if (!response.body) {
|
||||||
|
const text = await response.text()
|
||||||
|
if (!response.ok) throw parseResponseErrorText(text, response.status)
|
||||||
|
return JSON.parse(text) as T
|
||||||
|
}
|
||||||
|
|
||||||
|
const reader = response.body.getReader()
|
||||||
|
const decoder = new TextDecoder()
|
||||||
|
let text = ''
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
const chunk = await readChunkWithIdleTimeout(
|
||||||
|
reader,
|
||||||
|
init?.signal ?? undefined
|
||||||
|
)
|
||||||
|
if (chunk.done) break
|
||||||
|
text += decoder.decode(chunk.value, { stream: true })
|
||||||
|
}
|
||||||
|
text += decoder.decode()
|
||||||
|
} finally {
|
||||||
|
reader.releaseLock()
|
||||||
|
}
|
||||||
|
if (!response.ok) throw parseResponseErrorText(text, response.status)
|
||||||
|
return JSON.parse(text) as T
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout)
|
||||||
|
init?.signal?.removeEventListener('abort', onAbort)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readChunkWithIdleTimeout(
|
||||||
|
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||||
|
signal?: AbortSignal
|
||||||
|
): Promise<ReadableStreamReadResult<Uint8Array>> {
|
||||||
|
if (signal?.aborted) {
|
||||||
|
await reader.cancel().catch(() => {})
|
||||||
|
throw createJobCancelledError()
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let settled = false
|
||||||
|
const cleanup = () => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
signal?.removeEventListener('abort', onAbort)
|
||||||
|
}
|
||||||
|
const onAbort = () => {
|
||||||
|
if (settled) return
|
||||||
|
settled = true
|
||||||
|
cleanup()
|
||||||
|
void reader.cancel().catch(() => {})
|
||||||
|
reject(createJobCancelledError())
|
||||||
|
}
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
if (settled) return
|
||||||
|
settled = true
|
||||||
|
cleanup()
|
||||||
|
void reader.cancel().catch(() => {})
|
||||||
|
reject(new DOMException(
|
||||||
|
'Target response body was idle for too long',
|
||||||
|
'TimeoutError'
|
||||||
|
))
|
||||||
|
}, SITE_SYNC_RESPONSE_IDLE_TIMEOUT_MS)
|
||||||
|
signal?.addEventListener('abort', onAbort, { once: true })
|
||||||
|
|
||||||
|
reader.read().then(
|
||||||
|
(result) => {
|
||||||
|
if (settled) return
|
||||||
|
settled = true
|
||||||
|
cleanup()
|
||||||
|
resolve(result)
|
||||||
|
},
|
||||||
|
(error) => {
|
||||||
|
if (settled) return
|
||||||
|
settled = true
|
||||||
|
cleanup()
|
||||||
|
reject(error)
|
||||||
|
}
|
||||||
|
)
|
||||||
})
|
})
|
||||||
if (!response.ok) throw await responseError(response)
|
|
||||||
return await response.json() as T
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fetchSiteSyncJson<T>(
|
export async function fetchSiteSyncJson<T>(
|
||||||
url: string,
|
url: string,
|
||||||
token: string,
|
token: string,
|
||||||
init?: RequestInit,
|
init?: RequestInit,
|
||||||
signal?: AbortSignal
|
signal?: AbortSignal,
|
||||||
|
onStatus?: (status: SiteSyncRetryStatus) => void
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
return retrySiteSyncOperation(
|
return retrySiteSyncOperation(
|
||||||
() => fetchSiteSyncJsonOnce<T>(url, token, {
|
() => fetchSiteSyncJsonOnce<T>(url, token, {
|
||||||
...init,
|
...init,
|
||||||
signal: signal
|
|
||||||
? AbortSignal.any([
|
|
||||||
signal,
|
|
||||||
AbortSignal.timeout(SITE_SYNC_JSON_TIMEOUT_MS)
|
|
||||||
])
|
|
||||||
: AbortSignal.timeout(SITE_SYNC_JSON_TIMEOUT_MS)
|
|
||||||
}),
|
|
||||||
signal
|
signal
|
||||||
|
}),
|
||||||
|
signal,
|
||||||
|
onStatus
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,15 +416,18 @@ async function uploadObject(
|
|||||||
basename(key)
|
basename(key)
|
||||||
)
|
)
|
||||||
|
|
||||||
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
const response = await undiciFetch(
|
||||||
targetUrl(options.targetBaseUrl, '/api/site-sync/object'),
|
targetUrl(options.targetBaseUrl, '/api/site-sync/object'),
|
||||||
token,
|
|
||||||
{
|
{
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
body: form,
|
body: form,
|
||||||
signal: getJobOperationSignal(jobId, SITE_SYNC_OBJECT_TIMEOUT_MS)
|
headers: { 'Auth-Token': token },
|
||||||
}
|
signal: getJobAbortSignal(jobId),
|
||||||
|
dispatcher: agents.upload
|
||||||
|
} as Parameters<typeof undiciFetch>[1]
|
||||||
)
|
)
|
||||||
|
if (!response.ok) throw await responseError(response)
|
||||||
|
await response.json().catch(() => null)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function downloadSiteSyncObject(
|
async function downloadSiteSyncObject(
|
||||||
@@ -222,7 +436,7 @@ async function downloadSiteSyncObject(
|
|||||||
token: string,
|
token: string,
|
||||||
key: string
|
key: string
|
||||||
): Promise<Buffer> {
|
): Promise<Buffer> {
|
||||||
const response = await fetch(
|
const response = await undiciFetch(
|
||||||
targetUrl(
|
targetUrl(
|
||||||
options.targetBaseUrl,
|
options.targetBaseUrl,
|
||||||
`/api/site-sync/object?key=${encodeURIComponent(key)}`
|
`/api/site-sync/object?key=${encodeURIComponent(key)}`
|
||||||
@@ -231,7 +445,8 @@ async function downloadSiteSyncObject(
|
|||||||
headers: {
|
headers: {
|
||||||
'Auth-Token': token
|
'Auth-Token': token
|
||||||
},
|
},
|
||||||
signal: getJobOperationSignal(jobId, SITE_SYNC_OBJECT_TIMEOUT_MS)
|
signal: getJobAbortSignal(jobId),
|
||||||
|
dispatcher: agents.object
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
if (!response.ok) throw await responseError(response)
|
if (!response.ok) throw await responseError(response)
|
||||||
@@ -287,6 +502,16 @@ async function runSiteSyncPull(
|
|||||||
const progressMessage = (suffix = '') => (
|
const progressMessage = (suffix = '') => (
|
||||||
`扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}${suffix}`
|
`扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}${suffix}`
|
||||||
)
|
)
|
||||||
|
const updateLiveStatus = (step: string, status?: SiteSyncRetryStatus) => {
|
||||||
|
const detail = status ? formatRetryStatus(step, status) : step
|
||||||
|
try {
|
||||||
|
updateBackupJob(jobId, {
|
||||||
|
message: progressMessage(` · ${detail}`)
|
||||||
|
})
|
||||||
|
} catch {
|
||||||
|
// Progress messages must never fail the sync itself.
|
||||||
|
}
|
||||||
|
}
|
||||||
const failAndCheckpoint = (error: unknown) => {
|
const failAndCheckpoint = (error: unknown) => {
|
||||||
const cancelled = isJobCancellationRequested(jobId)
|
const cancelled = isJobCancellationRequested(jobId)
|
||||||
|| isJobCancelledError(error)
|
|| isJobCancelledError(error)
|
||||||
@@ -311,7 +536,8 @@ async function runSiteSyncPull(
|
|||||||
targetUrl(options.targetBaseUrl, '/api/site-sync/users'),
|
targetUrl(options.targetBaseUrl, '/api/site-sync/users'),
|
||||||
token,
|
token,
|
||||||
undefined,
|
undefined,
|
||||||
jobSignal
|
jobSignal,
|
||||||
|
status => updateLiveStatus('连接目标站点', status)
|
||||||
)).users
|
)).users
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -324,15 +550,18 @@ async function runSiteSyncPull(
|
|||||||
targetUrl(options.targetBaseUrl, path),
|
targetUrl(options.targetBaseUrl, path),
|
||||||
token,
|
token,
|
||||||
undefined,
|
undefined,
|
||||||
jobSignal
|
jobSignal,
|
||||||
|
status => updateLiveStatus('读取远程媒体清单', status)
|
||||||
)
|
)
|
||||||
|
|
||||||
const processBundleOnce = async (
|
const processBundleOnce = async (
|
||||||
bundle: SiteSyncMediaBundle,
|
bundle: SiteSyncMediaBundle,
|
||||||
isRetry: boolean
|
isRetry: boolean,
|
||||||
|
setStep: (step: string) => void
|
||||||
): Promise<'added' | 'skipped'> => {
|
): Promise<'added' | 'skipped'> => {
|
||||||
const { media, assets } = bundle
|
const { media, assets } = bundle
|
||||||
throwIfJobCancellationRequested(jobId)
|
throwIfJobCancellationRequested(jobId)
|
||||||
|
setStep(`准备同步媒体 ${media.key}`)
|
||||||
if (media.ownerUsername && !targetUsers.has(media.ownerUsername)) {
|
if (media.ownerUsername && !targetUsers.has(media.ownerUsername)) {
|
||||||
throw new Error(`当前站点缺少同名用户:${media.ownerUsername}`)
|
throw new Error(`当前站点缺少同名用户:${media.ownerUsername}`)
|
||||||
}
|
}
|
||||||
@@ -350,6 +579,7 @@ async function runSiteSyncPull(
|
|||||||
: null
|
: null
|
||||||
for (const asset of assets) {
|
for (const asset of assets) {
|
||||||
throwIfJobCancellationRequested(jobId)
|
throwIfJobCancellationRequested(jobId)
|
||||||
|
setStep(`下载对象 ${basename(asset.key)}`)
|
||||||
const existedBefore = Boolean(await backend.head(asset.key))
|
const existedBefore = Boolean(await backend.head(asset.key))
|
||||||
if (options.conflictMode === 'skip' && existedBefore) {
|
if (options.conflictMode === 'skip' && existedBefore) {
|
||||||
continue
|
continue
|
||||||
@@ -366,6 +596,7 @@ async function runSiteSyncPull(
|
|||||||
`同步对象大小不一致:${asset.key},预期 ${asset.size},实际 ${bytes.length}`
|
`同步对象大小不一致:${asset.key},预期 ${asset.size},实际 ${bytes.length}`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
setStep(`写入对象 ${basename(asset.key)}`)
|
||||||
const isPrimary = asset.key === media.key
|
const isPrimary = asset.key === media.key
|
||||||
recordSiteSyncArtifact({
|
recordSiteSyncArtifact({
|
||||||
jobId,
|
jobId,
|
||||||
@@ -393,16 +624,23 @@ async function runSiteSyncPull(
|
|||||||
throwIfJobCancellationRequested(jobId)
|
throwIfJobCancellationRequested(jobId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setStep(`写入媒体记录 ${media.key}`)
|
||||||
await applySiteSyncMedia(media, 'overwrite', actorUserId)
|
await applySiteSyncMedia(media, 'overwrite', actorUserId)
|
||||||
completeSiteSyncMediaArtifacts(jobId, media.key)
|
completeSiteSyncMediaArtifacts(jobId, media.key)
|
||||||
return 'added'
|
return 'added'
|
||||||
}
|
}
|
||||||
const processBundle = (bundle: SiteSyncMediaBundle, isRetry: boolean) => (
|
const processBundle = (bundle: SiteSyncMediaBundle, isRetry: boolean) => {
|
||||||
retrySiteSyncOperation(
|
let step = `同步媒体 ${bundle.media.key}`
|
||||||
() => processBundleOnce(bundle, isRetry),
|
return retrySiteSyncOperation(
|
||||||
jobSignal
|
() => processBundleOnce(
|
||||||
)
|
bundle,
|
||||||
|
isRetry,
|
||||||
|
(value) => { step = value }
|
||||||
|
),
|
||||||
|
jobSignal,
|
||||||
|
status => updateLiveStatus(step, status)
|
||||||
)
|
)
|
||||||
|
}
|
||||||
const recordOutcome = (outcome: 'added' | 'skipped', fileCount: number) => {
|
const recordOutcome = (outcome: 'added' | 'skipped', fileCount: number) => {
|
||||||
if (outcome === 'added') added += 1
|
if (outcome === 'added') added += 1
|
||||||
else skipped += 1
|
else skipped += 1
|
||||||
@@ -525,7 +763,16 @@ export async function runSiteSyncJob(
|
|||||||
targetUrl(options.targetBaseUrl, '/api/site-sync/inventory'),
|
targetUrl(options.targetBaseUrl, '/api/site-sync/inventory'),
|
||||||
token,
|
token,
|
||||||
undefined,
|
undefined,
|
||||||
jobSignal
|
jobSignal,
|
||||||
|
(status) => {
|
||||||
|
try {
|
||||||
|
updateBackupJob(jobId, {
|
||||||
|
message: formatRetryStatus('连接目标站点', status)
|
||||||
|
})
|
||||||
|
} catch {
|
||||||
|
// Keep the sync running even if a progress update cannot be stored.
|
||||||
|
}
|
||||||
|
}
|
||||||
)
|
)
|
||||||
const targetUsers = new Set(inventory.users)
|
const targetUsers = new Set(inventory.users)
|
||||||
const targetMedia = new Set(inventory.mediaKeys)
|
const targetMedia = new Set(inventory.mediaKeys)
|
||||||
@@ -614,6 +861,16 @@ export async function runSiteSyncJob(
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
const updateLiveStatus = (step: string, status?: SiteSyncRetryStatus) => {
|
||||||
|
const detail = status ? formatRetryStatus(step, status) : step
|
||||||
|
try {
|
||||||
|
updateBackupJob(jobId, {
|
||||||
|
message: `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · ${detail}`
|
||||||
|
})
|
||||||
|
} catch {
|
||||||
|
// Progress messages must never fail the sync itself.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
plans.sort((a, b) => (
|
plans.sort((a, b) => (
|
||||||
a.row.key < b.row.key
|
a.row.key < b.row.key
|
||||||
@@ -624,14 +881,20 @@ export async function runSiteSyncJob(
|
|||||||
))
|
))
|
||||||
lastMediaKey = lastMediaKey || ''
|
lastMediaKey = lastMediaKey || ''
|
||||||
let lastCheckpointAt = 0
|
let lastCheckpointAt = 0
|
||||||
const processPlanOnce = async (plan: typeof plans[number], isRetry: boolean) => {
|
const processPlanOnce = async (
|
||||||
|
plan: typeof plans[number],
|
||||||
|
isRetry: boolean,
|
||||||
|
setStep: (step: string) => void
|
||||||
|
) => {
|
||||||
const { row, ownerUsername, backend, keys } = plan
|
const { row, ownerUsername, backend, keys } = plan
|
||||||
throwIfJobCancellationRequested(jobId)
|
throwIfJobCancellationRequested(jobId)
|
||||||
|
setStep(`准备同步媒体 ${row.key}`)
|
||||||
if (
|
if (
|
||||||
!isRetry
|
!isRetry
|
||||||
&& options.conflictMode === 'skip'
|
&& options.conflictMode === 'skip'
|
||||||
&& targetMedia.has(row.key)
|
&& targetMedia.has(row.key)
|
||||||
) {
|
) {
|
||||||
|
setStep(`提交已存在的媒体 ${row.key}`)
|
||||||
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
||||||
targetUrl(options.targetBaseUrl, '/api/site-sync/complete'),
|
targetUrl(options.targetBaseUrl, '/api/site-sync/complete'),
|
||||||
token,
|
token,
|
||||||
@@ -639,7 +902,7 @@ export async function runSiteSyncJob(
|
|||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ jobId, mediaKey: row.key }),
|
body: JSON.stringify({ jobId, mediaKey: row.key }),
|
||||||
signal: getJobOperationSignal(jobId, SITE_SYNC_JSON_TIMEOUT_MS)
|
signal: getJobAbortSignal(jobId)
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
skipped += 1
|
skipped += 1
|
||||||
@@ -650,6 +913,7 @@ export async function runSiteSyncJob(
|
|||||||
|
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
throwIfJobCancellationRequested(jobId)
|
throwIfJobCancellationRequested(jobId)
|
||||||
|
setStep(`上传对象 ${basename(key)}`)
|
||||||
const stored = await backend.head(key)
|
const stored = await backend.head(key)
|
||||||
if (!stored) throw new Error(`源文件不存在:${key}`)
|
if (!stored) throw new Error(`源文件不存在:${key}`)
|
||||||
if (
|
if (
|
||||||
@@ -670,6 +934,7 @@ export async function runSiteSyncJob(
|
|||||||
}
|
}
|
||||||
|
|
||||||
throwIfJobCancellationRequested(jobId)
|
throwIfJobCancellationRequested(jobId)
|
||||||
|
setStep(`写入媒体记录 ${row.key}`)
|
||||||
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
||||||
targetUrl(options.targetBaseUrl, '/api/site-sync/media'),
|
targetUrl(options.targetBaseUrl, '/api/site-sync/media'),
|
||||||
token,
|
token,
|
||||||
@@ -681,7 +946,7 @@ export async function runSiteSyncJob(
|
|||||||
media: toSiteSyncMediaTransferWithTags(row, ownerUsername),
|
media: toSiteSyncMediaTransferWithTags(row, ownerUsername),
|
||||||
conflictMode: options.conflictMode
|
conflictMode: options.conflictMode
|
||||||
}),
|
}),
|
||||||
signal: getJobOperationSignal(jobId, SITE_SYNC_JSON_TIMEOUT_MS)
|
signal: getJobAbortSignal(jobId)
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -692,10 +957,18 @@ export async function runSiteSyncJob(
|
|||||||
const processPlan = (
|
const processPlan = (
|
||||||
plan: typeof plans[number],
|
plan: typeof plans[number],
|
||||||
isRetry: boolean
|
isRetry: boolean
|
||||||
) => retrySiteSyncOperation(
|
) => {
|
||||||
() => processPlanOnce(plan, isRetry),
|
let step = `同步媒体 ${plan.row.key}`
|
||||||
jobSignal
|
return retrySiteSyncOperation(
|
||||||
|
() => processPlanOnce(
|
||||||
|
plan,
|
||||||
|
isRetry,
|
||||||
|
(value) => { step = value }
|
||||||
|
),
|
||||||
|
jobSignal,
|
||||||
|
status => updateLiveStatus(step, status)
|
||||||
)
|
)
|
||||||
|
}
|
||||||
|
|
||||||
const retryPlans = plans.filter(plan => failedKeys.has(plan.row.key))
|
const retryPlans = plans.filter(plan => failedKeys.has(plan.row.key))
|
||||||
for (const plan of retryPlans) {
|
for (const plan of retryPlans) {
|
||||||
@@ -712,6 +985,8 @@ export async function runSiteSyncJob(
|
|||||||
jobId,
|
jobId,
|
||||||
cancelled
|
cancelled
|
||||||
? '任务已取消'
|
? '任务已取消'
|
||||||
|
: error instanceof Error
|
||||||
|
? `${error.message} · 已保存断点`
|
||||||
: '连续 3 次重试失败,已保存断点',
|
: '连续 3 次重试失败,已保存断点',
|
||||||
message
|
message
|
||||||
)
|
)
|
||||||
@@ -735,6 +1010,8 @@ export async function runSiteSyncJob(
|
|||||||
jobId,
|
jobId,
|
||||||
cancelled
|
cancelled
|
||||||
? '任务已取消'
|
? '任务已取消'
|
||||||
|
: error instanceof Error
|
||||||
|
? `${error.message} · 已保存断点`
|
||||||
: '连续 3 次重试失败,已保存断点',
|
: '连续 3 次重试失败,已保存断点',
|
||||||
message
|
message
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in new issue
Block a user