站点同步

This commit is contained in:
tianmo committed 2026-10-17 14:41:32 +08:00
1 parent a64be50b73
commit 9cb8552a99
28 files changed
+1683 -18

No files matched your search

+31 -1
View File
@@ -1,5 +1,5 @@
import type { H3Event } from 'h3'
import { findUserByApiToken, getPrimaryAdminUserId } from './db'
import { findUserById, findUserByApiToken, getPrimaryAdminUserId } from './db'
import { getAdminSecret, getApiUploadToken } from './env'
import { createApiError } from './api-error'
import {
@@ -110,6 +110,36 @@ export function resolveUserIdFromApiToken(
return null
}
/** 两站点同步:全站 Token 或管理员个人 Token;也允许管理员网页会话。 */
export async function requireSiteSyncAuth(event: H3Event): Promise<AuthUser> {
if (devBypass(event)) {
return { id: 0, username: 'dev', role: 'admin' }
}
if (verifyGlobalApiUploadToken(event)) {
const adminId = getPrimaryAdminUserId()
if (adminId != null) {
const admin = findUserById(adminId)
if (admin) {
return { id: admin.id, username: admin.username, role: admin.role }
}
}
}
const userId = resolveUserIdFromApiToken(event)
if (userId != null) {
const user = findUserById(userId)
if (user?.role === 'admin') {
return { id: user.id, username: user.username, role: user.role }
}
}
const sessionUser = await getCurrentUser(event)
if (sessionUser?.role === 'admin') return sessionUser
createApiError(event, 'FORBIDDEN', '需要目标站点管理员同步 Token', 403)
}
export function verifyApiUploadToken(event: H3Event): boolean {
return verifyGlobalApiUploadToken(event)
|| resolveUserIdFromApiToken(event) !== null
+2
View File
@@ -9,6 +9,8 @@ export {
startFullExportJob,
startFullRestoreJob,
startBackendSyncJob,
startSiteSyncJob,
resumeSiteSyncJob,
isBackupRunnerBusy,
cancelBackupJob
} from './runner'
+1 -1
View File
@@ -2,7 +2,7 @@ import { randomBytes } from 'node:crypto'
import { getDb, getPrimaryAdminUserId } from '../db'
import { ensureBackupSchema } from './schema'
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync'
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
export interface BackupJob {
+53
View File
@@ -1,6 +1,7 @@
import { runBackendSyncJob } from './backend-sync'
import { runFullExportJob } from './full-export'
import { runFullRestoreJob } from './full-restore'
import { registerSiteSyncToken, runSiteSyncJob, type SiteSyncOptions } from '../site-sync/runner'
import {
createBackupJob,
getBackupJob,
@@ -114,3 +115,55 @@ export function startBackendSyncJob(options: {
scheduleBackupJob(job, () => runBackendSyncJob(job.id, options))
return getBackupJob(job.id)!
}
export function startSiteSyncJob(
options: SiteSyncOptions,
token: string,
userId?: number | null
): BackupJob {
claimJobSlot()
const job = createBackupJob({
type: 'site_sync',
userId,
message: '等待开始…',
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: options.direction
}
})
registerSiteSyncToken(job.id, token)
scheduleBackupJob(job, () => runSiteSyncJob(job.id, options))
return getBackupJob(job.id)!
}
export function resumeSiteSyncJob(
jobId: string,
token: string,
userId?: number | null
): BackupJob {
claimJobSlot()
const job = getBackupJob(jobId, userId ?? undefined)
if (
!job
|| job.type !== 'site_sync'
|| job.status !== 'failed'
|| job.payload.resumable !== true
) {
throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
}
const targetBaseUrl = typeof job.payload.targetBaseUrl === 'string'
? job.payload.targetBaseUrl
: ''
const conflictMode = job.payload.conflictMode === 'skip' ? 'skip' : 'overwrite'
const direction = job.payload.direction === 'pull' ? 'pull' : 'push'
if (!targetBaseUrl) throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
registerSiteSyncToken(job.id, token)
scheduleBackupJob(job, () => runSiteSyncJob(job.id, {
targetBaseUrl,
conflictMode,
direction
}))
return getBackupJob(job.id)!
}
+41 -1
View File
@@ -1,11 +1,50 @@
import { getDb } from '../db'
function backupJobsTableSql(): string {
const row = getDb().prepare(`
SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'backup_jobs'
`).get() as { sql?: string } | undefined
return row?.sql ?? ''
}
function migrateBackupJobTypes(): void {
if (backupJobsTableSql().includes('\'site_sync\'')) return
const db = getDb()
db.exec(`
CREATE TABLE backup_jobs_new (
id TEXT PRIMARY KEY,
user_id INTEGER,
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
progress INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0,
message TEXT NOT NULL DEFAULT '',
payload_json TEXT NOT NULL DEFAULT '{}',
output_path TEXT,
error TEXT,
created_at TEXT NOT NULL,
finished_at TEXT
);
INSERT INTO backup_jobs_new
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
SELECT id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at
FROM backup_jobs;
DROP TABLE backup_jobs;
ALTER TABLE backup_jobs_new RENAME TO backup_jobs;
CREATE INDEX IF NOT EXISTS idx_backup_jobs_created_at
ON backup_jobs(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_backup_jobs_user_created
ON backup_jobs(user_id, created_at DESC);
`)
}
export function ensureBackupSchema(): void {
getDb().exec(`
CREATE TABLE IF NOT EXISTS backup_jobs (
id TEXT PRIMARY KEY,
user_id INTEGER,
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync')),
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
progress INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0,
@@ -36,4 +75,5 @@ export function ensureBackupSchema(): void {
WHERE user_id IS NULL
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
`)
migrateBackupJobTypes()
}
+23 -7
View File
@@ -35,6 +35,7 @@ export interface ImageIndexRow {
backend_id: string
user_id: number | null
folder: string
media_type: MediaType
original_name: string
content_type: string
size: number
@@ -71,6 +72,7 @@ function rowToStoredImage(row: ImageIndexRow): StoredImage {
contentType: row.content_type,
size: row.size,
userId: row.user_id,
mediaType: row.media_type,
hlsKey: row.hls_key ?? null,
transcodeStatus: (row.transcode_status ?? 'none') as TranscodeStatus,
transcodeProgress: Number(row.transcode_progress ?? 0),
@@ -232,7 +234,7 @@ export async function purgeOrphanImageIndexRows(): Promise<number> {
export function getImageIndexRow(key: string): ImageIndexRow | null {
ensureStorageSchema()
const row = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE key = ?
`).get(key) as ImageIndexRow | undefined
@@ -425,16 +427,30 @@ export function getImageTotals(): { count: number, bytes: number } {
export function listAllImageIndexRows(): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
ORDER BY key ASC
`).all() as unknown as ImageIndexRow[]
}
export function listImageIndexRowsAfter(
afterKey: string,
limit: number
): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE key > ?
ORDER BY key ASC
LIMIT ?
`).all(afterKey, limit) as unknown as ImageIndexRow[]
}
export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE user_id = ?
ORDER BY key ASC
@@ -444,7 +460,7 @@ export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
export function listImageIndexRowsByBackend(backendId: string): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE backend_id = ?
ORDER BY key ASC
@@ -687,7 +703,7 @@ export async function listImages(options: {
const backendSql = buildBackendFilterSql(options.backendId, params)
const tagSql = buildTagFilterSql(options.tagFilter, params)
const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
ORDER BY uploaded_at DESC, key DESC
@@ -733,7 +749,7 @@ export async function searchImages(options: {
const backendSql = buildBackendFilterSql(options.backendId, params)
const tagSql = buildTagFilterSql(options.tagFilter, params)
const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
ORDER BY uploaded_at DESC, key DESC
@@ -867,7 +883,7 @@ export async function getUserScopedStorageStats(
export async function listImageKeysForAutoDelete(): Promise<StoredImage[]> {
ensureStorageSchema()
const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE media_type = 'image'
ORDER BY uploaded_at ASC
+1
View File
@@ -8,6 +8,7 @@ const row: ImageIndexRow = {
backend_id: 'local',
user_id: 1,
folder: 'videos',
media_type: 'video',
original_name: 'example.mp4',
content_type: 'video/mp4',
size: 100,
+79
View File
@@ -0,0 +1,79 @@
import { findUserByUsername } from '../db'
import {
getImageIndexRow,
insertImageIndex,
updateImageThumb,
updateImageTranscode
} from '../image-index'
import { getActiveBackend } from '../storage/resolver'
import {
createTag,
findTagByName,
setImageTags
} from '../tags'
import type { SiteSyncMediaTransfer } from './protocol'
export async function applySiteSyncMedia(
media: SiteSyncMediaTransfer,
conflictMode: 'skip' | 'overwrite',
actorUserId: number
): Promise<{ success: true, skipped: boolean }> {
if (conflictMode === 'skip' && getImageIndexRow(media.key)) {
return { success: true, skipped: true }
}
const owner = media.ownerUsername
? findUserByUsername(media.ownerUsername)
: null
if (media.ownerUsername && !owner) {
throw new Error(`当前站点缺少用户:${media.ownerUsername}`)
}
const backend = await getActiveBackend()
const requiredKeys = [
...(!media.sourceDeleted ? [media.key] : []),
...(media.thumbKey ? [media.thumbKey] : []),
...(media.hlsKey ? [media.hlsKey] : [])
]
for (const key of requiredKeys) {
if (!await backend.head(key)) {
throw new Error(`同步对象尚未写入:${key}`)
}
}
insertImageIndex({
key: media.key,
backendId: backend.id,
userId: owner?.id ?? null,
originalName: media.originalName,
contentType: media.contentType,
size: media.size,
uploadedAt: media.uploadedAt,
mediaType: media.mediaType
})
if (media.thumbKey) updateImageThumb(media.key, media.thumbKey)
if (media.mediaType === 'video') {
updateImageTranscode(media.key, {
status: media.transcodeStatus,
hlsKey: media.hlsKey,
progress: media.transcodeProgress,
preset: media.transcodePreset,
sourceDeleted: media.sourceDeleted,
error: media.transcodeError
})
}
const tagOwnerId = owner?.id ?? actorUserId
const tagIds: number[] = []
for (const tag of media.tags) {
const existing = findTagByName(tagOwnerId, tag.name)
const item = existing
? { id: existing.id }
: createTag(tagOwnerId, tag.name, tag.color)
tagIds.push(item.id)
}
setImageTags(media.key, tagIds, tagOwnerId, true)
return { success: true, skipped: false }
}
+38
View File
@@ -0,0 +1,38 @@
import type { ImageIndexRow } from '../image-index'
import { getTagsForImageKeys } from '../tags'
import type {
SiteSyncMediaTransfer,
SiteSyncTag
} from './protocol'
export function toSiteSyncMediaTransfer(
row: ImageIndexRow,
ownerUsername: string | null,
tags: SiteSyncTag[]
): SiteSyncMediaTransfer {
return {
key: row.key,
ownerUsername,
originalName: row.original_name,
contentType: row.content_type,
size: row.size,
uploadedAt: row.uploaded_at,
mediaType: row.media_type,
thumbKey: row.thumb_key,
hlsKey: row.hls_key,
transcodeStatus: row.transcode_status as SiteSyncMediaTransfer['transcodeStatus'],
transcodeProgress: Number(row.transcode_progress ?? 0),
transcodePreset: row.transcode_preset,
transcodeError: row.transcode_error,
sourceDeleted: Number(row.source_deleted ?? 0) === 1,
tags
}
}
export function toSiteSyncMediaTransferWithTags(
row: ImageIndexRow,
ownerUsername: string | null
): SiteSyncMediaTransfer {
const tags = getTagsForImageKeys([row.key]).get(row.key) ?? []
return toSiteSyncMediaTransfer(row, ownerUsername, tags)
}
+59
View File
@@ -0,0 +1,59 @@
import type { MediaType, TranscodeStatus } from '../storage/types'
export interface SiteSyncTag {
name: string
color: string
}
export interface SiteSyncMediaTransfer {
key: string
ownerUsername: string | null
originalName: string
contentType: string
size: number
uploadedAt: string
mediaType: MediaType
thumbKey: string | null
hlsKey: string | null
transcodeStatus: TranscodeStatus
transcodeProgress: number
transcodePreset: string | null
transcodeError: string | null
sourceDeleted: boolean
tags: SiteSyncTag[]
}
export interface SiteSyncInventory {
users: string[]
mediaKeys: string[]
objects: Array<{ key: string, size: number }>
}
export interface SiteSyncMediaAsset {
key: string
size: number
}
export interface SiteSyncMediaBundle {
media: SiteSyncMediaTransfer
assets: SiteSyncMediaAsset[]
}
export interface SiteSyncMediaPage {
items: SiteSyncMediaBundle[]
totalMedia: number
nextCursor: string | null
}
export interface SiteSyncUsers {
users: string[]
}
export type SiteSyncDirection = 'push' | 'pull'
export interface SiteSyncObjectOptions {
targetBaseUrl: string
token: string
conflictMode: 'skip' | 'overwrite'
direction?: SiteSyncDirection
}
+55
View File
@@ -0,0 +1,55 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { fetchSiteSyncJson, retrySiteSyncOperation } from './runner'
afterEach(() => {
vi.unstubAllGlobals()
})
describe('fetchSiteSyncJson', () => {
it('retries a broken connection up to three attempts', async () => {
const fetchMock = vi.fn()
.mockRejectedValueOnce(new Error('connection reset'))
.mockRejectedValueOnce(new Error('connection reset'))
.mockResolvedValueOnce(new Response('{"ok":true}', {
status: 200,
headers: { 'Content-Type': 'application/json' }
}))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).resolves.toEqual({ ok: true })
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('retries client errors because every sync failure gets three attempts', async () => {
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
new Response('{"message":"invalid"}', {
status: 400,
headers: { 'Content-Type': 'application/json' }
})
))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('invalid')
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('fails after three server errors', async () => {
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
new Response('{"message":"unavailable"}', {
status: 503,
headers: { 'Content-Type': 'application/json' }
})
))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('unavailable')
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('retries local sync failures and stops after the third attempt', async () => {
const operation = vi.fn().mockRejectedValue(new Error('database is busy'))
await expect(retrySiteSyncOperation(operation)).rejects.toThrow('database is busy')
expect(operation).toHaveBeenCalledTimes(3)
})
})
+641
View File
@@ -0,0 +1,641 @@
import { basename } from 'node:path'
import { contentTypeFromKey } from '../content-type'
import {
getImageIndexRow,
listAllImageIndexRows,
type ImageIndexRow
} from '../image-index'
import {
findUserByUsername,
getPrimaryAdminUserId,
listUserIdUsernameMap
} from '../db'
import { collectMediaAssetKeys } from '../media-assets'
import {
getActiveBackend,
getBackendForKey
} from '../storage/resolver'
import type { StorageBackend } from '../storage/types'
import {
getBackupJob,
markJobDone,
markJobFailed,
updateBackupJob
} from '../backup/jobs'
import { applySiteSyncMedia } from './apply'
import { toSiteSyncMediaTransferWithTags } from './manifest'
import type {
SiteSyncDirection,
SiteSyncInventory,
SiteSyncMediaBundle,
SiteSyncMediaPage,
SiteSyncUsers
} from './protocol'
export interface SiteSyncOptions {
targetBaseUrl: string
conflictMode: 'skip' | 'overwrite'
direction: SiteSyncDirection
}
const tokens = (globalThis as typeof globalThis & {
__pichostSiteSyncTokens?: Map<string, string>
}).__pichostSiteSyncTokens ?? new Map<string, string>()
;(globalThis as typeof globalThis & {
__pichostSiteSyncTokens?: Map<string, string>
}).__pichostSiteSyncTokens = tokens
export function registerSiteSyncToken(jobId: string, token: string): void {
tokens.set(jobId, token)
}
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = []
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
}
return Buffer.concat(chunks)
}
function targetUrl(baseUrl: string, path: string): string {
return `${baseUrl.replace(/\/+$/, '')}${path}`
}
async function responseError(response: Response): Promise<Error> {
const payload = await response.json().catch(() => null) as {
message?: string
statusMessage?: string
} | null
return new Error(payload?.message || payload?.statusMessage || `目标站点返回 ${response.status}`)
}
const SITE_SYNC_MAX_ATTEMPTS = 3
export async function retrySiteSyncOperation<T>(
operation: () => Promise<T>
): Promise<T> {
let lastError: Error | null = null
for (let attempt = 1; attempt <= SITE_SYNC_MAX_ATTEMPTS; attempt++) {
try {
return await operation()
} catch (error) {
lastError = error instanceof Error ? error : new Error('两站点同步失败')
if (attempt < SITE_SYNC_MAX_ATTEMPTS) {
await new Promise(resolve => setTimeout(resolve, attempt * 500))
}
}
}
throw lastError ?? new Error('两站点同步失败')
}
async function fetchSiteSyncJsonOnce<T>(
url: string,
token: string,
init?: RequestInit
): Promise<T> {
const response = await fetch(url, {
...init,
headers: {
'Auth-Token': token,
...(init?.headers ?? {})
}
})
if (!response.ok) throw await responseError(response)
return await response.json() as T
}
export async function fetchSiteSyncJson<T>(
url: string,
token: string,
init?: RequestInit
): Promise<T> {
return retrySiteSyncOperation(() => (
fetchSiteSyncJsonOnce<T>(url, token, init)
))
}
async function uploadObject(
options: SiteSyncOptions,
token: string,
backend: StorageBackend,
row: ImageIndexRow,
ownerUsername: string | null,
key: string
): Promise<void> {
const stored = await backend.head(key)
if (!stored) throw new Error(`源文件不存在:${key}`)
const bytes = await streamToBuffer(await backend.createStream(key))
const form = new FormData()
const isPrimary = key === row.key
form.append('key', key)
form.append('conflictMode', options.conflictMode)
form.append('ownerUsername', ownerUsername ?? '')
form.append('originalName', isPrimary ? row.original_name : basename(key))
form.append('uploadedAt', row.uploaded_at)
const contentType = isPrimary ? row.content_type : contentTypeFromKey(key)
form.append('contentType', contentType)
form.append(
'file',
new Blob([new Uint8Array(bytes)], { type: contentType }),
basename(key)
)
await fetchSiteSyncJsonOnce<{ success: boolean }>(
targetUrl(options.targetBaseUrl, '/api/site-sync/object'),
token,
{ method: 'POST', body: form }
)
}
async function downloadSiteSyncObject(
options: SiteSyncOptions,
token: string,
key: string
): Promise<Buffer> {
const response = await fetch(
targetUrl(
options.targetBaseUrl,
`/api/site-sync/object?key=${encodeURIComponent(key)}`
),
{
headers: {
'Auth-Token': token
}
}
)
if (!response.ok) throw await responseError(response)
return Buffer.from(await response.arrayBuffer())
}
async function runSiteSyncPull(
jobId: string,
options: SiteSyncOptions,
token: string
): Promise<void> {
const job = getBackupJob(jobId)
const checkpoint = job?.payload ?? {}
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
? checkpoint.lastMediaKey
: ''
let scanned = Number(checkpoint.scanned ?? 0)
let added = Number(checkpoint.added ?? 0)
let skipped = Number(checkpoint.skipped ?? 0)
let filesSynced = Number(checkpoint.filesSynced ?? 0)
let totalFiles = Number(checkpoint.totalFiles ?? 0)
let totalMedia = Number(checkpoint.totalMedia ?? job?.total ?? 0)
const failedKeys = new Set(
Array.isArray(checkpoint.failedKeys)
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
: []
)
const saveCheckpoint = (message: string) => {
updateBackupJob(jobId, {
progress: scanned,
total: totalMedia,
message,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'pull',
lastMediaKey,
scanned,
added,
skipped,
failed: failedKeys.size,
filesSynced,
totalFiles,
totalMedia,
failedKeys: [...failedKeys],
resumable: true
}
})
}
const progressMessage = (suffix = '') => (
`扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}${suffix}`
)
const failAndCheckpoint = (error: unknown) => {
const message = progressMessage(' · 已保存断点')
saveCheckpoint(message)
markJobFailed(
jobId,
error instanceof Error ? error.message : '从目标站点拉取失败',
message
)
}
try {
const targetUsers = new Set(
(await fetchSiteSyncJson<SiteSyncUsers>(
targetUrl(options.targetBaseUrl, '/api/site-sync/users'),
token
)).users
)
const localUserMap = listUserIdUsernameMap()
const localUsers = new Set(localUserMap.values())
const missingUsers = [...targetUsers].filter(name => !localUsers.has(name))
if (missingUsers.length) {
throw new Error(`当前站点缺少同名用户:${missingUsers.join('、')}`)
}
const backend = await retrySiteSyncOperation(() => getActiveBackend())
const actorUserId = job?.userId ?? getPrimaryAdminUserId() ?? 0
const readMediaPage = (path: string) => fetchSiteSyncJson<SiteSyncMediaPage>(
targetUrl(options.targetBaseUrl, path),
token
)
const processBundleOnce = async (
bundle: SiteSyncMediaBundle,
isRetry: boolean
): Promise<'added' | 'skipped'> => {
const { media, assets } = bundle
if (
!isRetry
&& options.conflictMode === 'skip'
&& getImageIndexRow(media.key)
) {
return 'skipped'
}
const owner = media.ownerUsername
? findUserByUsername(media.ownerUsername)
: null
for (const asset of assets) {
if (
options.conflictMode === 'skip'
&& await backend.head(asset.key)
) {
continue
}
const bytes = await downloadSiteSyncObject(
options,
token,
asset.key
)
if (bytes.length !== asset.size) {
throw new Error(
`同步对象大小不一致:${asset.key},预期 ${asset.size},实际 ${bytes.length}`
)
}
const isPrimary = asset.key === media.key
await backend.put(asset.key, new Uint8Array(bytes), {
originalName: isPrimary ? media.originalName : basename(asset.key),
uploadedAt: media.uploadedAt,
contentType: isPrimary
? media.contentType
: contentTypeFromKey(asset.key),
size: bytes.length,
userId: owner?.id ?? null,
mediaType: isPrimary
? media.mediaType
: asset.key.startsWith('videos/')
? 'video'
: asset.key.startsWith('files/')
? 'file'
: 'image'
})
}
await applySiteSyncMedia(media, 'overwrite', actorUserId)
return 'added'
}
const processBundle = (bundle: SiteSyncMediaBundle, isRetry: boolean) => (
retrySiteSyncOperation(() => processBundleOnce(bundle, isRetry))
)
const recordOutcome = (outcome: 'added' | 'skipped', fileCount: number) => {
if (outcome === 'added') added += 1
else skipped += 1
filesSynced += fileCount
totalFiles = Math.max(totalFiles, filesSynced)
}
for (const key of [...failedKeys]) {
try {
const page = await readMediaPage(
`/api/site-sync/media?key=${encodeURIComponent(key)}`
)
const bundle = page.items[0]
if (!bundle) throw new Error(`目标媒体不存在:${key}`)
const outcome = await processBundle(bundle, true)
recordOutcome(outcome, bundle.assets.length)
failedKeys.delete(key)
if (key > lastMediaKey) lastMediaKey = key
saveCheckpoint(progressMessage())
} catch (error) {
failedKeys.add(key)
failAndCheckpoint(error)
return
}
}
let cursor = lastMediaKey
let lastCheckpointAt = 0
while (true) {
let page: SiteSyncMediaPage
try {
page = await readMediaPage(
`/api/site-sync/media?limit=50${cursor ? `&after=${encodeURIComponent(cursor)}` : ''}`
)
} catch (error) {
failAndCheckpoint(error)
return
}
totalMedia = Math.max(totalMedia, page.totalMedia)
for (const bundle of page.items) {
scanned += 1
try {
const outcome = await processBundle(bundle, false)
recordOutcome(outcome, bundle.assets.length)
failedKeys.delete(bundle.media.key)
} catch (error) {
failedKeys.add(bundle.media.key)
lastMediaKey = bundle.media.key
failAndCheckpoint(error)
return
}
lastMediaKey = bundle.media.key
const now = Date.now()
if (
now - lastCheckpointAt >= 1000
|| scanned === totalMedia
|| !page.nextCursor
) {
saveCheckpoint(progressMessage())
lastCheckpointAt = now
}
}
if (!page.nextCursor) break
cursor = page.nextCursor
}
if (failedKeys.size) {
markJobFailed(
jobId,
`${failedKeys.size} 条媒体拉取失败`,
progressMessage()
)
saveCheckpoint(progressMessage())
return
}
totalFiles = filesSynced
markJobDone(jobId, {
message: `扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'pull',
lastMediaKey,
scanned,
added,
skipped,
failed: 0,
filesSynced,
totalFiles,
totalMedia: totalMedia || scanned,
failedKeys: [],
resumable: false
}
})
} catch (error) {
failAndCheckpoint(error)
}
}
export async function runSiteSyncJob(
jobId: string,
options: SiteSyncOptions
): Promise<void> {
const token = tokens.get(jobId)
if (!token) throw new Error('同步 Token 不可用,请重新发起任务')
try {
if (options.direction === 'pull') {
await runSiteSyncPull(jobId, options, token)
return
}
const inventory = await fetchSiteSyncJson<SiteSyncInventory>(
targetUrl(options.targetBaseUrl, '/api/site-sync/inventory'),
token
)
const targetUsers = new Set(inventory.users)
const targetMedia = new Set(inventory.mediaKeys)
const targetObjects = new Map(inventory.objects.map(item => [item.key, item.size]))
const rows = listAllImageIndexRows()
const ownerMap = listUserIdUsernameMap()
const job = getBackupJob(jobId)
const checkpoint = job?.payload ?? {}
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
? checkpoint.lastMediaKey
: ''
let scanned = Number(checkpoint.scanned ?? 0)
let added = Number(checkpoint.added ?? 0)
let skipped = Number(checkpoint.skipped ?? 0)
let filesSynced = Number(checkpoint.filesSynced ?? 0)
const failedKeys = new Set(
Array.isArray(checkpoint.failedKeys)
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
: []
)
const missingUsers = [...new Set(
rows
.map(row => row.user_id == null ? null : ownerMap.get(row.user_id))
.filter((name): name is string => Boolean(name))
)].filter(name => !targetUsers.has(name))
if (missingUsers.length) {
throw new Error(`目标站点缺少同名用户:${missingUsers.join('、')}`)
}
const plans: Array<{
row: ImageIndexRow
ownerUsername: string | null
backend: StorageBackend
keys: string[]
}> = []
let totalFiles = 0
updateBackupJob(jobId, {
total: rows.length,
progress: scanned,
message: '正在统计同步文件…'
})
for (const row of rows) {
const backend = await getBackendForKey(row.key)
let assetKeys: string[]
try {
assetKeys = await collectMediaAssetKeys(row, backend)
} catch {
assetKeys = [
...(!Number(row.source_deleted ?? 0) ? [row.key] : []),
...(row.thumb_key ? [row.thumb_key] : []),
...(row.hls_key ? [row.hls_key] : [])
]
}
const keys = [...new Set(assetKeys)]
totalFiles += keys.length
plans.push({
row,
ownerUsername: row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
backend,
keys
})
}
const saveCheckpoint = (message: string) => {
updateBackupJob(jobId, {
progress: scanned,
total: rows.length,
message,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'push',
lastMediaKey,
scanned,
added,
skipped,
failed: failedKeys.size,
filesSynced,
totalFiles,
failedKeys: [...failedKeys],
resumable: true
}
})
}
plans.sort((a, b) => (
a.row.key < b.row.key
? -1
: a.row.key > b.row.key
? 1
: 0
))
lastMediaKey = lastMediaKey || ''
let lastCheckpointAt = 0
const processPlanOnce = async (plan: typeof plans[number], isRetry: boolean) => {
const { row, ownerUsername, backend, keys } = plan
if (
!isRetry
&& options.conflictMode === 'skip'
&& targetMedia.has(row.key)
) {
skipped += 1
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
failedKeys.delete(row.key)
return
}
for (const key of keys) {
const stored = await backend.head(key)
if (!stored) throw new Error(`源文件不存在:${key}`)
if (
options.conflictMode === 'skip'
&& targetObjects.get(key) === stored.size
) {
continue
}
await uploadObject(options, token, backend, row, ownerUsername, key)
}
await fetchSiteSyncJsonOnce<{ success: boolean }>(
targetUrl(options.targetBaseUrl, '/api/site-sync/media'),
token,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
media: toSiteSyncMediaTransferWithTags(row, ownerUsername),
conflictMode: options.conflictMode
})
}
)
failedKeys.delete(row.key)
added += 1
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
}
const processPlan = (
plan: typeof plans[number],
isRetry: boolean
) => retrySiteSyncOperation(() => processPlanOnce(plan, isRetry))
const retryPlans = plans.filter(plan => failedKeys.has(plan.row.key))
for (const plan of retryPlans) {
try {
await processPlan(plan, true)
lastMediaKey = plan.row.key
} catch {
failedKeys.add(plan.row.key)
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
saveCheckpoint(message)
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
return
}
}
for (const plan of plans) {
if (plan.row.key <= lastMediaKey) continue
scanned += 1
try {
await processPlan(plan, false)
} catch {
failedKeys.add(plan.row.key)
lastMediaKey = plan.row.key
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
saveCheckpoint(message)
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
return
}
lastMediaKey = plan.row.key
const now = Date.now()
if (now - lastCheckpointAt >= 1000 || scanned === rows.length || failedKeys.size) {
saveCheckpoint(
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
lastCheckpointAt = now
}
}
if (failedKeys.size) {
markJobFailed(
jobId,
`${failedKeys.size} 条媒体同步失败`,
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
saveCheckpoint(
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
return
}
markJobDone(jobId, {
message: `扫描 ${rows.length}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'push',
lastMediaKey,
scanned: rows.length,
added,
skipped,
failed: 0,
filesSynced: totalFiles,
totalFiles,
failedKeys: [],
resumable: false
}
})
} catch (error) {
const message = error instanceof Error ? error.message : '两站点同步失败'
markJobFailed(jobId, message)
throw error
} finally {
tokens.delete(jobId)
}
}