站点同步
This commit is contained in:
1 parent
a64be50b73
commit
9cb8552a99
28 files changed
+1683
-18
No files matched your search
+31
-1
@@ -1,5 +1,5 @@
|
||||
import type { H3Event } from 'h3'
|
||||
import { findUserByApiToken, getPrimaryAdminUserId } from './db'
|
||||
import { findUserById, findUserByApiToken, getPrimaryAdminUserId } from './db'
|
||||
import { getAdminSecret, getApiUploadToken } from './env'
|
||||
import { createApiError } from './api-error'
|
||||
import {
|
||||
@@ -110,6 +110,36 @@ export function resolveUserIdFromApiToken(
|
||||
return null
|
||||
}
|
||||
|
||||
/** 两站点同步:全站 Token 或管理员个人 Token;也允许管理员网页会话。 */
|
||||
export async function requireSiteSyncAuth(event: H3Event): Promise<AuthUser> {
|
||||
if (devBypass(event)) {
|
||||
return { id: 0, username: 'dev', role: 'admin' }
|
||||
}
|
||||
|
||||
if (verifyGlobalApiUploadToken(event)) {
|
||||
const adminId = getPrimaryAdminUserId()
|
||||
if (adminId != null) {
|
||||
const admin = findUserById(adminId)
|
||||
if (admin) {
|
||||
return { id: admin.id, username: admin.username, role: admin.role }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const userId = resolveUserIdFromApiToken(event)
|
||||
if (userId != null) {
|
||||
const user = findUserById(userId)
|
||||
if (user?.role === 'admin') {
|
||||
return { id: user.id, username: user.username, role: user.role }
|
||||
}
|
||||
}
|
||||
|
||||
const sessionUser = await getCurrentUser(event)
|
||||
if (sessionUser?.role === 'admin') return sessionUser
|
||||
|
||||
createApiError(event, 'FORBIDDEN', '需要目标站点管理员同步 Token', 403)
|
||||
}
|
||||
|
||||
export function verifyApiUploadToken(event: H3Event): boolean {
|
||||
return verifyGlobalApiUploadToken(event)
|
||||
|| resolveUserIdFromApiToken(event) !== null
|
||||
|
||||
@@ -9,6 +9,8 @@ export {
|
||||
startFullExportJob,
|
||||
startFullRestoreJob,
|
||||
startBackendSyncJob,
|
||||
startSiteSyncJob,
|
||||
resumeSiteSyncJob,
|
||||
isBackupRunnerBusy,
|
||||
cancelBackupJob
|
||||
} from './runner'
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomBytes } from 'node:crypto'
|
||||
import { getDb, getPrimaryAdminUserId } from '../db'
|
||||
import { ensureBackupSchema } from './schema'
|
||||
|
||||
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync'
|
||||
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
|
||||
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
|
||||
|
||||
export interface BackupJob {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { runBackendSyncJob } from './backend-sync'
|
||||
import { runFullExportJob } from './full-export'
|
||||
import { runFullRestoreJob } from './full-restore'
|
||||
import { registerSiteSyncToken, runSiteSyncJob, type SiteSyncOptions } from '../site-sync/runner'
|
||||
import {
|
||||
createBackupJob,
|
||||
getBackupJob,
|
||||
@@ -114,3 +115,55 @@ export function startBackendSyncJob(options: {
|
||||
scheduleBackupJob(job, () => runBackendSyncJob(job.id, options))
|
||||
return getBackupJob(job.id)!
|
||||
}
|
||||
|
||||
export function startSiteSyncJob(
|
||||
options: SiteSyncOptions,
|
||||
token: string,
|
||||
userId?: number | null
|
||||
): BackupJob {
|
||||
claimJobSlot()
|
||||
const job = createBackupJob({
|
||||
type: 'site_sync',
|
||||
userId,
|
||||
message: '等待开始…',
|
||||
payload: {
|
||||
targetBaseUrl: options.targetBaseUrl,
|
||||
conflictMode: options.conflictMode,
|
||||
direction: options.direction
|
||||
}
|
||||
})
|
||||
registerSiteSyncToken(job.id, token)
|
||||
scheduleBackupJob(job, () => runSiteSyncJob(job.id, options))
|
||||
return getBackupJob(job.id)!
|
||||
}
|
||||
|
||||
export function resumeSiteSyncJob(
|
||||
jobId: string,
|
||||
token: string,
|
||||
userId?: number | null
|
||||
): BackupJob {
|
||||
claimJobSlot()
|
||||
const job = getBackupJob(jobId, userId ?? undefined)
|
||||
if (
|
||||
!job
|
||||
|| job.type !== 'site_sync'
|
||||
|| job.status !== 'failed'
|
||||
|| job.payload.resumable !== true
|
||||
) {
|
||||
throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
||||
}
|
||||
const targetBaseUrl = typeof job.payload.targetBaseUrl === 'string'
|
||||
? job.payload.targetBaseUrl
|
||||
: ''
|
||||
const conflictMode = job.payload.conflictMode === 'skip' ? 'skip' : 'overwrite'
|
||||
const direction = job.payload.direction === 'pull' ? 'pull' : 'push'
|
||||
if (!targetBaseUrl) throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
||||
|
||||
registerSiteSyncToken(job.id, token)
|
||||
scheduleBackupJob(job, () => runSiteSyncJob(job.id, {
|
||||
targetBaseUrl,
|
||||
conflictMode,
|
||||
direction
|
||||
}))
|
||||
return getBackupJob(job.id)!
|
||||
}
|
||||
@@ -1,11 +1,50 @@
|
||||
import { getDb } from '../db'
|
||||
|
||||
function backupJobsTableSql(): string {
|
||||
const row = getDb().prepare(`
|
||||
SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'backup_jobs'
|
||||
`).get() as { sql?: string } | undefined
|
||||
return row?.sql ?? ''
|
||||
}
|
||||
|
||||
function migrateBackupJobTypes(): void {
|
||||
if (backupJobsTableSql().includes('\'site_sync\'')) return
|
||||
|
||||
const db = getDb()
|
||||
db.exec(`
|
||||
CREATE TABLE backup_jobs_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id INTEGER,
|
||||
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
|
||||
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
output_path TEXT,
|
||||
error TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
finished_at TEXT
|
||||
);
|
||||
INSERT INTO backup_jobs_new
|
||||
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
|
||||
SELECT id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at
|
||||
FROM backup_jobs;
|
||||
DROP TABLE backup_jobs;
|
||||
ALTER TABLE backup_jobs_new RENAME TO backup_jobs;
|
||||
CREATE INDEX IF NOT EXISTS idx_backup_jobs_created_at
|
||||
ON backup_jobs(created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_backup_jobs_user_created
|
||||
ON backup_jobs(user_id, created_at DESC);
|
||||
`)
|
||||
}
|
||||
|
||||
export function ensureBackupSchema(): void {
|
||||
getDb().exec(`
|
||||
CREATE TABLE IF NOT EXISTS backup_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id INTEGER,
|
||||
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync')),
|
||||
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
|
||||
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -36,4 +75,5 @@ export function ensureBackupSchema(): void {
|
||||
WHERE user_id IS NULL
|
||||
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
|
||||
`)
|
||||
migrateBackupJobTypes()
|
||||
}
|
||||
@@ -35,6 +35,7 @@ export interface ImageIndexRow {
|
||||
backend_id: string
|
||||
user_id: number | null
|
||||
folder: string
|
||||
media_type: MediaType
|
||||
original_name: string
|
||||
content_type: string
|
||||
size: number
|
||||
@@ -71,6 +72,7 @@ function rowToStoredImage(row: ImageIndexRow): StoredImage {
|
||||
contentType: row.content_type,
|
||||
size: row.size,
|
||||
userId: row.user_id,
|
||||
mediaType: row.media_type,
|
||||
hlsKey: row.hls_key ?? null,
|
||||
transcodeStatus: (row.transcode_status ?? 'none') as TranscodeStatus,
|
||||
transcodeProgress: Number(row.transcode_progress ?? 0),
|
||||
@@ -232,7 +234,7 @@ export async function purgeOrphanImageIndexRows(): Promise<number> {
|
||||
export function getImageIndexRow(key: string): ImageIndexRow | null {
|
||||
ensureStorageSchema()
|
||||
const row = getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
WHERE key = ?
|
||||
`).get(key) as ImageIndexRow | undefined
|
||||
@@ -425,16 +427,30 @@ export function getImageTotals(): { count: number, bytes: number } {
|
||||
export function listAllImageIndexRows(): ImageIndexRow[] {
|
||||
ensureStorageSchema()
|
||||
return getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
ORDER BY key ASC
|
||||
`).all() as unknown as ImageIndexRow[]
|
||||
}
|
||||
|
||||
export function listImageIndexRowsAfter(
|
||||
afterKey: string,
|
||||
limit: number
|
||||
): ImageIndexRow[] {
|
||||
ensureStorageSchema()
|
||||
return getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
WHERE key > ?
|
||||
ORDER BY key ASC
|
||||
LIMIT ?
|
||||
`).all(afterKey, limit) as unknown as ImageIndexRow[]
|
||||
}
|
||||
|
||||
export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
|
||||
ensureStorageSchema()
|
||||
return getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
WHERE user_id = ?
|
||||
ORDER BY key ASC
|
||||
@@ -444,7 +460,7 @@ export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
|
||||
export function listImageIndexRowsByBackend(backendId: string): ImageIndexRow[] {
|
||||
ensureStorageSchema()
|
||||
return getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
WHERE backend_id = ?
|
||||
ORDER BY key ASC
|
||||
@@ -687,7 +703,7 @@ export async function listImages(options: {
|
||||
const backendSql = buildBackendFilterSql(options.backendId, params)
|
||||
const tagSql = buildTagFilterSql(options.tagFilter, params)
|
||||
const rows = getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
|
||||
ORDER BY uploaded_at DESC, key DESC
|
||||
@@ -733,7 +749,7 @@ export async function searchImages(options: {
|
||||
const backendSql = buildBackendFilterSql(options.backendId, params)
|
||||
const tagSql = buildTagFilterSql(options.tagFilter, params)
|
||||
const rows = getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
|
||||
ORDER BY uploaded_at DESC, key DESC
|
||||
@@ -867,7 +883,7 @@ export async function getUserScopedStorageStats(
|
||||
export async function listImageKeysForAutoDelete(): Promise<StoredImage[]> {
|
||||
ensureStorageSchema()
|
||||
const rows = getDb().prepare(`
|
||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||
FROM images
|
||||
WHERE media_type = 'image'
|
||||
ORDER BY uploaded_at ASC
|
||||
|
||||
@@ -8,6 +8,7 @@ const row: ImageIndexRow = {
|
||||
backend_id: 'local',
|
||||
user_id: 1,
|
||||
folder: 'videos',
|
||||
media_type: 'video',
|
||||
original_name: 'example.mp4',
|
||||
content_type: 'video/mp4',
|
||||
size: 100,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { findUserByUsername } from '../db'
|
||||
import {
|
||||
getImageIndexRow,
|
||||
insertImageIndex,
|
||||
updateImageThumb,
|
||||
updateImageTranscode
|
||||
} from '../image-index'
|
||||
import { getActiveBackend } from '../storage/resolver'
|
||||
import {
|
||||
createTag,
|
||||
findTagByName,
|
||||
setImageTags
|
||||
} from '../tags'
|
||||
import type { SiteSyncMediaTransfer } from './protocol'
|
||||
|
||||
export async function applySiteSyncMedia(
|
||||
media: SiteSyncMediaTransfer,
|
||||
conflictMode: 'skip' | 'overwrite',
|
||||
actorUserId: number
|
||||
): Promise<{ success: true, skipped: boolean }> {
|
||||
if (conflictMode === 'skip' && getImageIndexRow(media.key)) {
|
||||
return { success: true, skipped: true }
|
||||
}
|
||||
|
||||
const owner = media.ownerUsername
|
||||
? findUserByUsername(media.ownerUsername)
|
||||
: null
|
||||
if (media.ownerUsername && !owner) {
|
||||
throw new Error(`当前站点缺少用户:${media.ownerUsername}`)
|
||||
}
|
||||
|
||||
const backend = await getActiveBackend()
|
||||
const requiredKeys = [
|
||||
...(!media.sourceDeleted ? [media.key] : []),
|
||||
...(media.thumbKey ? [media.thumbKey] : []),
|
||||
...(media.hlsKey ? [media.hlsKey] : [])
|
||||
]
|
||||
for (const key of requiredKeys) {
|
||||
if (!await backend.head(key)) {
|
||||
throw new Error(`同步对象尚未写入:${key}`)
|
||||
}
|
||||
}
|
||||
|
||||
insertImageIndex({
|
||||
key: media.key,
|
||||
backendId: backend.id,
|
||||
userId: owner?.id ?? null,
|
||||
originalName: media.originalName,
|
||||
contentType: media.contentType,
|
||||
size: media.size,
|
||||
uploadedAt: media.uploadedAt,
|
||||
mediaType: media.mediaType
|
||||
})
|
||||
|
||||
if (media.thumbKey) updateImageThumb(media.key, media.thumbKey)
|
||||
if (media.mediaType === 'video') {
|
||||
updateImageTranscode(media.key, {
|
||||
status: media.transcodeStatus,
|
||||
hlsKey: media.hlsKey,
|
||||
progress: media.transcodeProgress,
|
||||
preset: media.transcodePreset,
|
||||
sourceDeleted: media.sourceDeleted,
|
||||
error: media.transcodeError
|
||||
})
|
||||
}
|
||||
|
||||
const tagOwnerId = owner?.id ?? actorUserId
|
||||
const tagIds: number[] = []
|
||||
for (const tag of media.tags) {
|
||||
const existing = findTagByName(tagOwnerId, tag.name)
|
||||
const item = existing
|
||||
? { id: existing.id }
|
||||
: createTag(tagOwnerId, tag.name, tag.color)
|
||||
tagIds.push(item.id)
|
||||
}
|
||||
setImageTags(media.key, tagIds, tagOwnerId, true)
|
||||
|
||||
return { success: true, skipped: false }
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { ImageIndexRow } from '../image-index'
|
||||
import { getTagsForImageKeys } from '../tags'
|
||||
import type {
|
||||
SiteSyncMediaTransfer,
|
||||
SiteSyncTag
|
||||
} from './protocol'
|
||||
|
||||
export function toSiteSyncMediaTransfer(
|
||||
row: ImageIndexRow,
|
||||
ownerUsername: string | null,
|
||||
tags: SiteSyncTag[]
|
||||
): SiteSyncMediaTransfer {
|
||||
return {
|
||||
key: row.key,
|
||||
ownerUsername,
|
||||
originalName: row.original_name,
|
||||
contentType: row.content_type,
|
||||
size: row.size,
|
||||
uploadedAt: row.uploaded_at,
|
||||
mediaType: row.media_type,
|
||||
thumbKey: row.thumb_key,
|
||||
hlsKey: row.hls_key,
|
||||
transcodeStatus: row.transcode_status as SiteSyncMediaTransfer['transcodeStatus'],
|
||||
transcodeProgress: Number(row.transcode_progress ?? 0),
|
||||
transcodePreset: row.transcode_preset,
|
||||
transcodeError: row.transcode_error,
|
||||
sourceDeleted: Number(row.source_deleted ?? 0) === 1,
|
||||
tags
|
||||
}
|
||||
}
|
||||
|
||||
export function toSiteSyncMediaTransferWithTags(
|
||||
row: ImageIndexRow,
|
||||
ownerUsername: string | null
|
||||
): SiteSyncMediaTransfer {
|
||||
const tags = getTagsForImageKeys([row.key]).get(row.key) ?? []
|
||||
return toSiteSyncMediaTransfer(row, ownerUsername, tags)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { MediaType, TranscodeStatus } from '../storage/types'
|
||||
|
||||
export interface SiteSyncTag {
|
||||
name: string
|
||||
color: string
|
||||
}
|
||||
|
||||
export interface SiteSyncMediaTransfer {
|
||||
key: string
|
||||
ownerUsername: string | null
|
||||
originalName: string
|
||||
contentType: string
|
||||
size: number
|
||||
uploadedAt: string
|
||||
mediaType: MediaType
|
||||
thumbKey: string | null
|
||||
hlsKey: string | null
|
||||
transcodeStatus: TranscodeStatus
|
||||
transcodeProgress: number
|
||||
transcodePreset: string | null
|
||||
transcodeError: string | null
|
||||
sourceDeleted: boolean
|
||||
tags: SiteSyncTag[]
|
||||
}
|
||||
|
||||
export interface SiteSyncInventory {
|
||||
users: string[]
|
||||
mediaKeys: string[]
|
||||
objects: Array<{ key: string, size: number }>
|
||||
}
|
||||
|
||||
export interface SiteSyncMediaAsset {
|
||||
key: string
|
||||
size: number
|
||||
}
|
||||
|
||||
export interface SiteSyncMediaBundle {
|
||||
media: SiteSyncMediaTransfer
|
||||
assets: SiteSyncMediaAsset[]
|
||||
}
|
||||
|
||||
export interface SiteSyncMediaPage {
|
||||
items: SiteSyncMediaBundle[]
|
||||
totalMedia: number
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
export interface SiteSyncUsers {
|
||||
users: string[]
|
||||
}
|
||||
|
||||
export type SiteSyncDirection = 'push' | 'pull'
|
||||
|
||||
export interface SiteSyncObjectOptions {
|
||||
targetBaseUrl: string
|
||||
token: string
|
||||
conflictMode: 'skip' | 'overwrite'
|
||||
direction?: SiteSyncDirection
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { fetchSiteSyncJson, retrySiteSyncOperation } from './runner'
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe('fetchSiteSyncJson', () => {
|
||||
it('retries a broken connection up to three attempts', async () => {
|
||||
const fetchMock = vi.fn()
|
||||
.mockRejectedValueOnce(new Error('connection reset'))
|
||||
.mockRejectedValueOnce(new Error('connection reset'))
|
||||
.mockResolvedValueOnce(new Response('{"ok":true}', {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
}))
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
await expect(fetchSiteSyncJson('http://target/api', 'token')).resolves.toEqual({ ok: true })
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('retries client errors because every sync failure gets three attempts', async () => {
|
||||
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
|
||||
new Response('{"message":"invalid"}', {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
})
|
||||
))
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('invalid')
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('fails after three server errors', async () => {
|
||||
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
|
||||
new Response('{"message":"unavailable"}', {
|
||||
status: 503,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
})
|
||||
))
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('unavailable')
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('retries local sync failures and stops after the third attempt', async () => {
|
||||
const operation = vi.fn().mockRejectedValue(new Error('database is busy'))
|
||||
|
||||
await expect(retrySiteSyncOperation(operation)).rejects.toThrow('database is busy')
|
||||
expect(operation).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,641 @@
|
||||
import { basename } from 'node:path'
|
||||
import { contentTypeFromKey } from '../content-type'
|
||||
import {
|
||||
getImageIndexRow,
|
||||
listAllImageIndexRows,
|
||||
type ImageIndexRow
|
||||
} from '../image-index'
|
||||
import {
|
||||
findUserByUsername,
|
||||
getPrimaryAdminUserId,
|
||||
listUserIdUsernameMap
|
||||
} from '../db'
|
||||
import { collectMediaAssetKeys } from '../media-assets'
|
||||
import {
|
||||
getActiveBackend,
|
||||
getBackendForKey
|
||||
} from '../storage/resolver'
|
||||
import type { StorageBackend } from '../storage/types'
|
||||
import {
|
||||
getBackupJob,
|
||||
markJobDone,
|
||||
markJobFailed,
|
||||
updateBackupJob
|
||||
} from '../backup/jobs'
|
||||
import { applySiteSyncMedia } from './apply'
|
||||
import { toSiteSyncMediaTransferWithTags } from './manifest'
|
||||
import type {
|
||||
SiteSyncDirection,
|
||||
SiteSyncInventory,
|
||||
SiteSyncMediaBundle,
|
||||
SiteSyncMediaPage,
|
||||
SiteSyncUsers
|
||||
} from './protocol'
|
||||
|
||||
export interface SiteSyncOptions {
|
||||
targetBaseUrl: string
|
||||
conflictMode: 'skip' | 'overwrite'
|
||||
direction: SiteSyncDirection
|
||||
}
|
||||
|
||||
const tokens = (globalThis as typeof globalThis & {
|
||||
__pichostSiteSyncTokens?: Map<string, string>
|
||||
}).__pichostSiteSyncTokens ?? new Map<string, string>()
|
||||
;(globalThis as typeof globalThis & {
|
||||
__pichostSiteSyncTokens?: Map<string, string>
|
||||
}).__pichostSiteSyncTokens = tokens
|
||||
|
||||
export function registerSiteSyncToken(jobId: string, token: string): void {
|
||||
tokens.set(jobId, token)
|
||||
}
|
||||
|
||||
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
|
||||
}
|
||||
return Buffer.concat(chunks)
|
||||
}
|
||||
|
||||
function targetUrl(baseUrl: string, path: string): string {
|
||||
return `${baseUrl.replace(/\/+$/, '')}${path}`
|
||||
}
|
||||
|
||||
async function responseError(response: Response): Promise<Error> {
|
||||
const payload = await response.json().catch(() => null) as {
|
||||
message?: string
|
||||
statusMessage?: string
|
||||
} | null
|
||||
return new Error(payload?.message || payload?.statusMessage || `目标站点返回 ${response.status}`)
|
||||
}
|
||||
|
||||
const SITE_SYNC_MAX_ATTEMPTS = 3
|
||||
|
||||
export async function retrySiteSyncOperation<T>(
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> {
|
||||
let lastError: Error | null = null
|
||||
for (let attempt = 1; attempt <= SITE_SYNC_MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
return await operation()
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error('两站点同步失败')
|
||||
if (attempt < SITE_SYNC_MAX_ATTEMPTS) {
|
||||
await new Promise(resolve => setTimeout(resolve, attempt * 500))
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError ?? new Error('两站点同步失败')
|
||||
}
|
||||
|
||||
async function fetchSiteSyncJsonOnce<T>(
|
||||
url: string,
|
||||
token: string,
|
||||
init?: RequestInit
|
||||
): Promise<T> {
|
||||
const response = await fetch(url, {
|
||||
...init,
|
||||
headers: {
|
||||
'Auth-Token': token,
|
||||
...(init?.headers ?? {})
|
||||
}
|
||||
})
|
||||
if (!response.ok) throw await responseError(response)
|
||||
return await response.json() as T
|
||||
}
|
||||
|
||||
export async function fetchSiteSyncJson<T>(
|
||||
url: string,
|
||||
token: string,
|
||||
init?: RequestInit
|
||||
): Promise<T> {
|
||||
return retrySiteSyncOperation(() => (
|
||||
fetchSiteSyncJsonOnce<T>(url, token, init)
|
||||
))
|
||||
}
|
||||
|
||||
async function uploadObject(
|
||||
options: SiteSyncOptions,
|
||||
token: string,
|
||||
backend: StorageBackend,
|
||||
row: ImageIndexRow,
|
||||
ownerUsername: string | null,
|
||||
key: string
|
||||
): Promise<void> {
|
||||
const stored = await backend.head(key)
|
||||
if (!stored) throw new Error(`源文件不存在:${key}`)
|
||||
const bytes = await streamToBuffer(await backend.createStream(key))
|
||||
const form = new FormData()
|
||||
const isPrimary = key === row.key
|
||||
form.append('key', key)
|
||||
form.append('conflictMode', options.conflictMode)
|
||||
form.append('ownerUsername', ownerUsername ?? '')
|
||||
form.append('originalName', isPrimary ? row.original_name : basename(key))
|
||||
form.append('uploadedAt', row.uploaded_at)
|
||||
const contentType = isPrimary ? row.content_type : contentTypeFromKey(key)
|
||||
form.append('contentType', contentType)
|
||||
form.append(
|
||||
'file',
|
||||
new Blob([new Uint8Array(bytes)], { type: contentType }),
|
||||
basename(key)
|
||||
)
|
||||
|
||||
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
||||
targetUrl(options.targetBaseUrl, '/api/site-sync/object'),
|
||||
token,
|
||||
{ method: 'POST', body: form }
|
||||
)
|
||||
}
|
||||
|
||||
async function downloadSiteSyncObject(
|
||||
options: SiteSyncOptions,
|
||||
token: string,
|
||||
key: string
|
||||
): Promise<Buffer> {
|
||||
const response = await fetch(
|
||||
targetUrl(
|
||||
options.targetBaseUrl,
|
||||
`/api/site-sync/object?key=${encodeURIComponent(key)}`
|
||||
),
|
||||
{
|
||||
headers: {
|
||||
'Auth-Token': token
|
||||
}
|
||||
}
|
||||
)
|
||||
if (!response.ok) throw await responseError(response)
|
||||
return Buffer.from(await response.arrayBuffer())
|
||||
}
|
||||
|
||||
async function runSiteSyncPull(
|
||||
jobId: string,
|
||||
options: SiteSyncOptions,
|
||||
token: string
|
||||
): Promise<void> {
|
||||
const job = getBackupJob(jobId)
|
||||
const checkpoint = job?.payload ?? {}
|
||||
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
|
||||
? checkpoint.lastMediaKey
|
||||
: ''
|
||||
let scanned = Number(checkpoint.scanned ?? 0)
|
||||
let added = Number(checkpoint.added ?? 0)
|
||||
let skipped = Number(checkpoint.skipped ?? 0)
|
||||
let filesSynced = Number(checkpoint.filesSynced ?? 0)
|
||||
let totalFiles = Number(checkpoint.totalFiles ?? 0)
|
||||
let totalMedia = Number(checkpoint.totalMedia ?? job?.total ?? 0)
|
||||
const failedKeys = new Set(
|
||||
Array.isArray(checkpoint.failedKeys)
|
||||
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
|
||||
: []
|
||||
)
|
||||
|
||||
const saveCheckpoint = (message: string) => {
|
||||
updateBackupJob(jobId, {
|
||||
progress: scanned,
|
||||
total: totalMedia,
|
||||
message,
|
||||
payload: {
|
||||
targetBaseUrl: options.targetBaseUrl,
|
||||
conflictMode: options.conflictMode,
|
||||
direction: 'pull',
|
||||
lastMediaKey,
|
||||
scanned,
|
||||
added,
|
||||
skipped,
|
||||
failed: failedKeys.size,
|
||||
filesSynced,
|
||||
totalFiles,
|
||||
totalMedia,
|
||||
failedKeys: [...failedKeys],
|
||||
resumable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
const progressMessage = (suffix = '') => (
|
||||
`扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}${suffix}`
|
||||
)
|
||||
const failAndCheckpoint = (error: unknown) => {
|
||||
const message = progressMessage(' · 已保存断点')
|
||||
saveCheckpoint(message)
|
||||
markJobFailed(
|
||||
jobId,
|
||||
error instanceof Error ? error.message : '从目标站点拉取失败',
|
||||
message
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
const targetUsers = new Set(
|
||||
(await fetchSiteSyncJson<SiteSyncUsers>(
|
||||
targetUrl(options.targetBaseUrl, '/api/site-sync/users'),
|
||||
token
|
||||
)).users
|
||||
)
|
||||
const localUserMap = listUserIdUsernameMap()
|
||||
const localUsers = new Set(localUserMap.values())
|
||||
const missingUsers = [...targetUsers].filter(name => !localUsers.has(name))
|
||||
if (missingUsers.length) {
|
||||
throw new Error(`当前站点缺少同名用户:${missingUsers.join('、')}`)
|
||||
}
|
||||
|
||||
const backend = await retrySiteSyncOperation(() => getActiveBackend())
|
||||
const actorUserId = job?.userId ?? getPrimaryAdminUserId() ?? 0
|
||||
const readMediaPage = (path: string) => fetchSiteSyncJson<SiteSyncMediaPage>(
|
||||
targetUrl(options.targetBaseUrl, path),
|
||||
token
|
||||
)
|
||||
|
||||
const processBundleOnce = async (
|
||||
bundle: SiteSyncMediaBundle,
|
||||
isRetry: boolean
|
||||
): Promise<'added' | 'skipped'> => {
|
||||
const { media, assets } = bundle
|
||||
if (
|
||||
!isRetry
|
||||
&& options.conflictMode === 'skip'
|
||||
&& getImageIndexRow(media.key)
|
||||
) {
|
||||
return 'skipped'
|
||||
}
|
||||
|
||||
const owner = media.ownerUsername
|
||||
? findUserByUsername(media.ownerUsername)
|
||||
: null
|
||||
for (const asset of assets) {
|
||||
if (
|
||||
options.conflictMode === 'skip'
|
||||
&& await backend.head(asset.key)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const bytes = await downloadSiteSyncObject(
|
||||
options,
|
||||
token,
|
||||
asset.key
|
||||
)
|
||||
if (bytes.length !== asset.size) {
|
||||
throw new Error(
|
||||
`同步对象大小不一致:${asset.key},预期 ${asset.size},实际 ${bytes.length}`
|
||||
)
|
||||
}
|
||||
const isPrimary = asset.key === media.key
|
||||
await backend.put(asset.key, new Uint8Array(bytes), {
|
||||
originalName: isPrimary ? media.originalName : basename(asset.key),
|
||||
uploadedAt: media.uploadedAt,
|
||||
contentType: isPrimary
|
||||
? media.contentType
|
||||
: contentTypeFromKey(asset.key),
|
||||
size: bytes.length,
|
||||
userId: owner?.id ?? null,
|
||||
mediaType: isPrimary
|
||||
? media.mediaType
|
||||
: asset.key.startsWith('videos/')
|
||||
? 'video'
|
||||
: asset.key.startsWith('files/')
|
||||
? 'file'
|
||||
: 'image'
|
||||
})
|
||||
}
|
||||
|
||||
await applySiteSyncMedia(media, 'overwrite', actorUserId)
|
||||
return 'added'
|
||||
}
|
||||
const processBundle = (bundle: SiteSyncMediaBundle, isRetry: boolean) => (
|
||||
retrySiteSyncOperation(() => processBundleOnce(bundle, isRetry))
|
||||
)
|
||||
const recordOutcome = (outcome: 'added' | 'skipped', fileCount: number) => {
|
||||
if (outcome === 'added') added += 1
|
||||
else skipped += 1
|
||||
filesSynced += fileCount
|
||||
totalFiles = Math.max(totalFiles, filesSynced)
|
||||
}
|
||||
|
||||
for (const key of [...failedKeys]) {
|
||||
try {
|
||||
const page = await readMediaPage(
|
||||
`/api/site-sync/media?key=${encodeURIComponent(key)}`
|
||||
)
|
||||
const bundle = page.items[0]
|
||||
if (!bundle) throw new Error(`目标媒体不存在:${key}`)
|
||||
const outcome = await processBundle(bundle, true)
|
||||
recordOutcome(outcome, bundle.assets.length)
|
||||
failedKeys.delete(key)
|
||||
if (key > lastMediaKey) lastMediaKey = key
|
||||
saveCheckpoint(progressMessage())
|
||||
} catch (error) {
|
||||
failedKeys.add(key)
|
||||
failAndCheckpoint(error)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
let cursor = lastMediaKey
|
||||
let lastCheckpointAt = 0
|
||||
while (true) {
|
||||
let page: SiteSyncMediaPage
|
||||
try {
|
||||
page = await readMediaPage(
|
||||
`/api/site-sync/media?limit=50${cursor ? `&after=${encodeURIComponent(cursor)}` : ''}`
|
||||
)
|
||||
} catch (error) {
|
||||
failAndCheckpoint(error)
|
||||
return
|
||||
}
|
||||
totalMedia = Math.max(totalMedia, page.totalMedia)
|
||||
|
||||
for (const bundle of page.items) {
|
||||
scanned += 1
|
||||
try {
|
||||
const outcome = await processBundle(bundle, false)
|
||||
recordOutcome(outcome, bundle.assets.length)
|
||||
failedKeys.delete(bundle.media.key)
|
||||
} catch (error) {
|
||||
failedKeys.add(bundle.media.key)
|
||||
lastMediaKey = bundle.media.key
|
||||
failAndCheckpoint(error)
|
||||
return
|
||||
}
|
||||
|
||||
lastMediaKey = bundle.media.key
|
||||
const now = Date.now()
|
||||
if (
|
||||
now - lastCheckpointAt >= 1000
|
||||
|| scanned === totalMedia
|
||||
|| !page.nextCursor
|
||||
) {
|
||||
saveCheckpoint(progressMessage())
|
||||
lastCheckpointAt = now
|
||||
}
|
||||
}
|
||||
|
||||
if (!page.nextCursor) break
|
||||
cursor = page.nextCursor
|
||||
}
|
||||
|
||||
if (failedKeys.size) {
|
||||
markJobFailed(
|
||||
jobId,
|
||||
`${failedKeys.size} 条媒体拉取失败`,
|
||||
progressMessage()
|
||||
)
|
||||
saveCheckpoint(progressMessage())
|
||||
return
|
||||
}
|
||||
|
||||
totalFiles = filesSynced
|
||||
markJobDone(jobId, {
|
||||
message: `扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
|
||||
payload: {
|
||||
targetBaseUrl: options.targetBaseUrl,
|
||||
conflictMode: options.conflictMode,
|
||||
direction: 'pull',
|
||||
lastMediaKey,
|
||||
scanned,
|
||||
added,
|
||||
skipped,
|
||||
failed: 0,
|
||||
filesSynced,
|
||||
totalFiles,
|
||||
totalMedia: totalMedia || scanned,
|
||||
failedKeys: [],
|
||||
resumable: false
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
failAndCheckpoint(error)
|
||||
}
|
||||
}
|
||||
|
||||
export async function runSiteSyncJob(
|
||||
jobId: string,
|
||||
options: SiteSyncOptions
|
||||
): Promise<void> {
|
||||
const token = tokens.get(jobId)
|
||||
if (!token) throw new Error('同步 Token 不可用,请重新发起任务')
|
||||
|
||||
try {
|
||||
if (options.direction === 'pull') {
|
||||
await runSiteSyncPull(jobId, options, token)
|
||||
return
|
||||
}
|
||||
|
||||
const inventory = await fetchSiteSyncJson<SiteSyncInventory>(
|
||||
targetUrl(options.targetBaseUrl, '/api/site-sync/inventory'),
|
||||
token
|
||||
)
|
||||
const targetUsers = new Set(inventory.users)
|
||||
const targetMedia = new Set(inventory.mediaKeys)
|
||||
const targetObjects = new Map(inventory.objects.map(item => [item.key, item.size]))
|
||||
const rows = listAllImageIndexRows()
|
||||
const ownerMap = listUserIdUsernameMap()
|
||||
const job = getBackupJob(jobId)
|
||||
const checkpoint = job?.payload ?? {}
|
||||
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
|
||||
? checkpoint.lastMediaKey
|
||||
: ''
|
||||
let scanned = Number(checkpoint.scanned ?? 0)
|
||||
let added = Number(checkpoint.added ?? 0)
|
||||
let skipped = Number(checkpoint.skipped ?? 0)
|
||||
let filesSynced = Number(checkpoint.filesSynced ?? 0)
|
||||
const failedKeys = new Set(
|
||||
Array.isArray(checkpoint.failedKeys)
|
||||
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
|
||||
: []
|
||||
)
|
||||
|
||||
const missingUsers = [...new Set(
|
||||
rows
|
||||
.map(row => row.user_id == null ? null : ownerMap.get(row.user_id))
|
||||
.filter((name): name is string => Boolean(name))
|
||||
)].filter(name => !targetUsers.has(name))
|
||||
if (missingUsers.length) {
|
||||
throw new Error(`目标站点缺少同名用户:${missingUsers.join('、')}`)
|
||||
}
|
||||
|
||||
const plans: Array<{
|
||||
row: ImageIndexRow
|
||||
ownerUsername: string | null
|
||||
backend: StorageBackend
|
||||
keys: string[]
|
||||
}> = []
|
||||
let totalFiles = 0
|
||||
updateBackupJob(jobId, {
|
||||
total: rows.length,
|
||||
progress: scanned,
|
||||
message: '正在统计同步文件…'
|
||||
})
|
||||
|
||||
for (const row of rows) {
|
||||
const backend = await getBackendForKey(row.key)
|
||||
let assetKeys: string[]
|
||||
try {
|
||||
assetKeys = await collectMediaAssetKeys(row, backend)
|
||||
} catch {
|
||||
assetKeys = [
|
||||
...(!Number(row.source_deleted ?? 0) ? [row.key] : []),
|
||||
...(row.thumb_key ? [row.thumb_key] : []),
|
||||
...(row.hls_key ? [row.hls_key] : [])
|
||||
]
|
||||
}
|
||||
const keys = [...new Set(assetKeys)]
|
||||
totalFiles += keys.length
|
||||
plans.push({
|
||||
row,
|
||||
ownerUsername: row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
|
||||
backend,
|
||||
keys
|
||||
})
|
||||
}
|
||||
|
||||
const saveCheckpoint = (message: string) => {
|
||||
updateBackupJob(jobId, {
|
||||
progress: scanned,
|
||||
total: rows.length,
|
||||
message,
|
||||
payload: {
|
||||
targetBaseUrl: options.targetBaseUrl,
|
||||
conflictMode: options.conflictMode,
|
||||
direction: 'push',
|
||||
lastMediaKey,
|
||||
scanned,
|
||||
added,
|
||||
skipped,
|
||||
failed: failedKeys.size,
|
||||
filesSynced,
|
||||
totalFiles,
|
||||
failedKeys: [...failedKeys],
|
||||
resumable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
plans.sort((a, b) => (
|
||||
a.row.key < b.row.key
|
||||
? -1
|
||||
: a.row.key > b.row.key
|
||||
? 1
|
||||
: 0
|
||||
))
|
||||
lastMediaKey = lastMediaKey || ''
|
||||
let lastCheckpointAt = 0
|
||||
const processPlanOnce = async (plan: typeof plans[number], isRetry: boolean) => {
|
||||
const { row, ownerUsername, backend, keys } = plan
|
||||
if (
|
||||
!isRetry
|
||||
&& options.conflictMode === 'skip'
|
||||
&& targetMedia.has(row.key)
|
||||
) {
|
||||
skipped += 1
|
||||
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
|
||||
failedKeys.delete(row.key)
|
||||
return
|
||||
}
|
||||
|
||||
for (const key of keys) {
|
||||
const stored = await backend.head(key)
|
||||
if (!stored) throw new Error(`源文件不存在:${key}`)
|
||||
if (
|
||||
options.conflictMode === 'skip'
|
||||
&& targetObjects.get(key) === stored.size
|
||||
) {
|
||||
continue
|
||||
}
|
||||
await uploadObject(options, token, backend, row, ownerUsername, key)
|
||||
}
|
||||
|
||||
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
||||
targetUrl(options.targetBaseUrl, '/api/site-sync/media'),
|
||||
token,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
media: toSiteSyncMediaTransferWithTags(row, ownerUsername),
|
||||
conflictMode: options.conflictMode
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
failedKeys.delete(row.key)
|
||||
added += 1
|
||||
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
|
||||
}
|
||||
const processPlan = (
|
||||
plan: typeof plans[number],
|
||||
isRetry: boolean
|
||||
) => retrySiteSyncOperation(() => processPlanOnce(plan, isRetry))
|
||||
|
||||
const retryPlans = plans.filter(plan => failedKeys.has(plan.row.key))
|
||||
for (const plan of retryPlans) {
|
||||
try {
|
||||
await processPlan(plan, true)
|
||||
lastMediaKey = plan.row.key
|
||||
} catch {
|
||||
failedKeys.add(plan.row.key)
|
||||
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
|
||||
saveCheckpoint(message)
|
||||
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
for (const plan of plans) {
|
||||
if (plan.row.key <= lastMediaKey) continue
|
||||
scanned += 1
|
||||
try {
|
||||
await processPlan(plan, false)
|
||||
} catch {
|
||||
failedKeys.add(plan.row.key)
|
||||
lastMediaKey = plan.row.key
|
||||
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
|
||||
saveCheckpoint(message)
|
||||
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
|
||||
return
|
||||
}
|
||||
lastMediaKey = plan.row.key
|
||||
const now = Date.now()
|
||||
if (now - lastCheckpointAt >= 1000 || scanned === rows.length || failedKeys.size) {
|
||||
saveCheckpoint(
|
||||
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
|
||||
)
|
||||
lastCheckpointAt = now
|
||||
}
|
||||
}
|
||||
|
||||
if (failedKeys.size) {
|
||||
markJobFailed(
|
||||
jobId,
|
||||
`${failedKeys.size} 条媒体同步失败`,
|
||||
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
|
||||
)
|
||||
saveCheckpoint(
|
||||
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
markJobDone(jobId, {
|
||||
message: `扫描 ${rows.length}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
|
||||
payload: {
|
||||
targetBaseUrl: options.targetBaseUrl,
|
||||
conflictMode: options.conflictMode,
|
||||
direction: 'push',
|
||||
lastMediaKey,
|
||||
scanned: rows.length,
|
||||
added,
|
||||
skipped,
|
||||
failed: 0,
|
||||
filesSynced: totalFiles,
|
||||
totalFiles,
|
||||
failedKeys: [],
|
||||
resumable: false
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : '两站点同步失败'
|
||||
markJobFailed(jobId, message)
|
||||
throw error
|
||||
} finally {
|
||||
tokens.delete(jobId)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user