站点同步

This commit is contained in:
tianmo committed 2026-10-17 14:41:32 +08:00
1 parent a64be50b73
commit 9cb8552a99
28 files changed
+1683 -18

No files matched your search

+2 -1
View File
@@ -26,7 +26,8 @@ function typeLabel(type: BackupJobItem['type']): string {
const map: Record<BackupJobItem['type'], string> = {
full_export: t('storage.backup.jobTypeExport'),
full_restore: t('storage.backup.jobTypeRestore'),
backend_sync: t('storage.backup.jobTypeSync')
backend_sync: t('storage.backup.jobTypeSync'),
site_sync: t('storage.backup.jobTypeSiteSync')
}
return map[type]
}
+256
View File
@@ -0,0 +1,256 @@
<script setup lang="ts">
import type {
SiteSyncDirection,
SiteSyncRequest
} from '~/types/backup'
const props = defineProps<{
busy: boolean
disabled?: boolean
activeJob?: {
progress: number
total: number
message: string
payload: Record<string, unknown>
} | null
resumableJob?: {
id: string
payload: Record<string, unknown>
} | null
}>()
const emit = defineEmits<{
sync: [body: SiteSyncRequest]
}>()
const { t } = useI18n()
const targetBaseUrl = ref('')
const token = ref('')
const conflictMode = ref<'skip' | 'overwrite'>('skip')
const direction = ref<SiteSyncDirection>('push')
watch(() => props.resumableJob, (job) => {
if (!job) return
direction.value = job.payload.direction === 'pull' ? 'pull' : 'push'
if (typeof job.payload.targetBaseUrl === 'string') {
targetBaseUrl.value = job.payload.targetBaseUrl
}
}, { immediate: true })
const canSubmit = computed(() =>
!props.busy
&& !props.disabled
&& Boolean(targetBaseUrl.value.trim())
&& Boolean(token.value.trim())
)
const canResume = computed(() =>
!props.busy
&& !props.disabled
&& Boolean(props.resumableJob)
&& Boolean(token.value.trim())
)
const fileProgress = computed(() => {
const payload = props.activeJob?.payload
return {
synced: Number(payload?.filesSynced ?? 0),
total: Number(payload?.totalFiles ?? 0)
}
})
const isPull = computed(() => direction.value === 'pull')
const directionSubtitle = computed(() => (
isPull.value
? t('storage.backup.siteSyncPullSubtitle')
: t('storage.backup.siteSyncPushSubtitle')
))
const endpointLabel = computed(() => (
isPull.value
? t('storage.backup.siteSyncSource')
: t('storage.backup.siteSyncTarget')
))
const tokenLabel = computed(() => (
isPull.value
? t('storage.backup.siteSyncSourceToken')
: t('storage.backup.siteSyncToken')
))
const tokenPlaceholder = computed(() => (
isPull.value
? t('storage.backup.siteSyncSourceTokenPlaceholder')
: t('storage.backup.siteSyncTokenPlaceholder')
))
const startLabel = computed(() => (
isPull.value
? t('storage.backup.siteSyncStartPull')
: t('storage.backup.siteSyncStartPush')
))
function submit(resume = false) {
if (resume ? !canResume.value : !canSubmit.value) return
emit('sync', {
targetBaseUrl: targetBaseUrl.value.trim(),
token: token.value.trim(),
conflictMode: conflictMode.value,
direction: direction.value,
...(resume && props.resumableJob ? { resumeJobId: props.resumableJob.id } : {})
})
token.value = ''
}
function startNew() {
submit(false)
}
function resumeSync() {
submit(true)
}
function setConflictMode(mode: 'skip' | 'overwrite') {
conflictMode.value = mode
}
function setDirection(value: SiteSyncDirection) {
direction.value = value
}
</script>
<template>
<article class="rounded-xl border border-default bg-elevated p-4 sm:p-5">
<div class="flex items-start gap-3">
<div class="flex size-10 shrink-0 items-center justify-center rounded-lg bg-primary/10 text-primary">
<UIcon
name="i-lucide-server-cog"
class="size-5"
/>
</div>
<div class="min-w-0">
<h3 class="text-base font-semibold">
{{ t('storage.backup.siteSyncTitle') }}
</h3>
<p class="mt-0.5 text-sm leading-relaxed text-muted">
{{ directionSubtitle }}
</p>
</div>
</div>
<div class="mt-4 grid gap-3 lg:grid-cols-[minmax(0,1.2fr)_minmax(0,1fr)_auto] lg:items-end">
<div class="space-y-1.5">
<label class="text-xs text-muted">{{ endpointLabel }}</label>
<UInput
v-model="targetBaseUrl"
placeholder="https://target.example.com"
size="sm"
class="w-full"
:disabled="busy || disabled"
/>
</div>
<div class="space-y-1.5">
<label class="text-xs text-muted">{{ tokenLabel }}</label>
<UInput
v-model="token"
type="password"
:placeholder="tokenPlaceholder"
size="sm"
class="w-full"
:disabled="busy || disabled"
/>
</div>
<UButton
:label="startLabel"
:icon="isPull ? 'i-lucide-cloud-download' : 'i-lucide-cloud-upload'"
color="primary"
:loading="busy"
:disabled="!canSubmit"
@click="startNew"
/>
<UButton
v-if="resumableJob"
:label="t('storage.backup.siteSyncResume')"
icon="i-lucide-play"
color="warning"
:loading="busy"
:disabled="!canResume"
class="lg:col-start-3"
@click="resumeSync"
/>
</div>
<div class="mt-3 grid gap-3 sm:grid-cols-2">
<div class="flex flex-wrap items-center gap-2">
<span class="text-xs text-muted">{{ t('storage.backup.siteSyncDirection') }}</span>
<UButton
size="xs"
:variant="direction === 'push' ? 'solid' : 'ghost'"
:color="direction === 'push' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncPush')"
:disabled="busy || disabled"
@click="setDirection('push')"
/>
<UButton
size="xs"
:variant="direction === 'pull' ? 'solid' : 'ghost'"
:color="direction === 'pull' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncPull')"
:disabled="busy || disabled"
@click="setDirection('pull')"
/>
</div>
<div class="flex flex-wrap items-center gap-2 sm:justify-end">
<span class="text-xs text-muted">{{ t('storage.backup.siteSyncConflict') }}</span>
<UButton
size="xs"
:variant="conflictMode === 'skip' ? 'solid' : 'ghost'"
:color="conflictMode === 'skip' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncSkip')"
:disabled="busy || disabled"
@click="setConflictMode('skip')"
/>
<UButton
size="xs"
:variant="conflictMode === 'overwrite' ? 'solid' : 'ghost'"
:color="conflictMode === 'overwrite' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncOverwrite')"
:disabled="busy || disabled"
@click="setConflictMode('overwrite')"
/>
</div>
</div>
<UAlert
v-if="activeJob"
class="mt-4"
color="primary"
variant="subtle"
icon="i-lucide-refresh-cw"
:description="activeJob.message"
/>
<div
v-if="activeJob"
class="mt-3 grid gap-3 rounded-lg border border-default bg-default/40 p-3 sm:grid-cols-2"
>
<div>
<p class="text-xs text-muted">
{{ t('storage.backup.siteSyncScanProgress') }}
</p>
<p class="mt-1 font-mono text-sm font-medium">
{{ activeJob.message }}
</p>
</div>
<div>
<p class="text-xs text-muted">
{{ t('storage.backup.siteSyncFileProgress') }}
</p>
<p class="mt-1 font-mono text-sm font-medium">
{{ fileProgress.synced }} / {{ fileProgress.total }}
</p>
</div>
</div>
<BackupJobProgress
v-if="activeJob"
class="mt-2"
:message="activeJob.message"
:progress="activeJob.progress"
:total="activeJob.total"
/>
</article>
</template>
+26 -1
View File
@@ -1,5 +1,9 @@
<script setup lang="ts">
import type { BackupJobItem, SyncDraft } from '~/types/backup'
import type {
BackupJobItem,
SiteSyncRequest,
SyncDraft
} from '~/types/backup'
import type { StorageBackendItem } from '~/types/storage'
const props = defineProps<{
@@ -31,6 +35,7 @@ const {
confirmRestore,
previewSync,
startSync,
startSiteSync,
retryJob,
cancelJob,
downloadUrl
@@ -45,6 +50,14 @@ const activeJob = computed(() => getActiveJob())
const exportBusy = computed(() => activeJob.value?.type === 'full_export')
const restoreBusy = computed(() => activeJob.value?.type === 'full_restore')
const syncBusy = computed(() => activeJob.value?.type === 'backend_sync')
const siteSyncBusy = computed(() => activeJob.value?.type === 'site_sync')
const resumableSiteSyncJob = computed(() =>
jobs.value.find(job =>
job.type === 'site_sync'
&& job.status === 'failed'
&& job.payload.resumable === true
) ?? null
)
const anyJobBusy = computed(() => Boolean(activeJob.value))
onMounted(async () => {
@@ -94,6 +107,10 @@ async function onSync(body: {
await startSync(body)
}
async function onSiteSync(body: SiteSyncRequest) {
await startSiteSync(body)
}
function onDownload(jobId: string) {
window.open(downloadUrl(jobId), '_blank')
}
@@ -159,6 +176,14 @@ function onViewJob(job: BackupJobItem) {
/>
</div>
<StorageSiteSyncCard
:busy="siteSyncBusy"
:disabled="anyJobBusy && !siteSyncBusy"
:active-job="siteSyncBusy ? activeJob : null"
:resumable-job="resumableSiteSyncJob"
@sync="onSiteSync"
/>
<StorageBackupJobTable
:jobs="jobs"
:polling="polling"
+23 -1
View File
@@ -3,7 +3,8 @@ import type {
BackupJobItem,
BackupPackageItem,
BackupRestorePreview,
BackendSyncPreview
BackendSyncPreview,
SiteSyncRequest
} from '~/types/backup'
const jobs = ref<BackupJobItem[]>([])
@@ -255,6 +256,26 @@ export function useBackup() {
}
}
async function startSiteSync(body: SiteSyncRequest) {
try {
const data = await $fetch<{ job: BackupJobItem }>(
'/api/admin/site-sync',
{
method: 'POST',
credentials: 'include',
body
}
)
await refreshJobsFirstPage()
startPolling()
toast.add({ title: t('storage.backup.siteSyncStarted'), color: 'success' })
return data.job
} catch (error: unknown) {
handleError(error, t('storage.backup.siteSyncFailed'))
throw error
}
}
async function retryJob(jobId: string) {
try {
await $fetch(`/api/admin/backup/jobs/${jobId}/retry`, {
@@ -312,6 +333,7 @@ export function useBackup() {
confirmRestore,
previewSync,
startSync,
startSiteSync,
retryJob,
cancelJob,
downloadUrl
+10 -1
View File
@@ -1,5 +1,6 @@
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync'
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
export type SiteSyncDirection = 'push' | 'pull'
export interface BackupJobItem {
id: string
@@ -51,3 +52,11 @@ export interface SyncDraft {
fromBackendId: string
toBackendId: string
}
export interface SiteSyncRequest {
targetBaseUrl: string
token: string
conflictMode: 'skip' | 'overwrite'
direction: SiteSyncDirection
resumeJobId?: string
}
+25
View File
@@ -645,6 +645,30 @@
"conflictSkip": "Skip existing files",
"conflictOverwrite": "Overwrite existing files and database",
"syncTitle": "Cross-backend sync",
"siteSyncTitle": "Site-to-site sync",
"siteSyncSubtitle": "Sync media indexes, images, videos, files, thumbnails, HLS segments and tags to another PicHost. Matching usernames must already exist on the target.",
"siteSyncPushSubtitle": "Push media indexes, images, videos, files, thumbnails, HLS segments and tags to another PicHost. Matching usernames must already exist on the target.",
"siteSyncPullSubtitle": "Pull media indexes, images, videos, files, thumbnails, HLS segments and tags from another PicHost. Matching usernames must already exist on this site.",
"siteSyncTarget": "Target site URL",
"siteSyncSource": "Source site URL",
"siteSyncToken": "Target admin API token",
"siteSyncSourceToken": "Source admin API token",
"siteSyncTokenPlaceholder": "Enter target admin token",
"siteSyncSourceTokenPlaceholder": "Enter source admin token",
"siteSyncStart": "Start sync",
"siteSyncStartPush": "Start push",
"siteSyncStartPull": "Start pull",
"siteSyncDirection": "Sync direction:",
"siteSyncPush": "Push to target",
"siteSyncPull": "Pull from target",
"siteSyncConflict": "Conflict policy:",
"siteSyncSkip": "Skip existing",
"siteSyncOverwrite": "Overwrite existing",
"siteSyncResume": "Resume last sync",
"siteSyncScanProgress": "Media scan progress",
"siteSyncFileProgress": "Synced file count",
"siteSyncStarted": "Site-to-site sync started",
"siteSyncFailed": "Failed to start site-to-site sync",
"syncMigrateTitle": "Cross-backend migration",
"syncMigrateSubtitle": "Migrate files between configured backends",
"syncSubtitle": "Copy files between configured backends and update the index",
@@ -694,6 +718,7 @@
"jobTypeExport": "Site export",
"jobTypeRestore": "Site restore",
"jobTypeSync": "Cross-backend sync",
"jobTypeSiteSync": "Site-to-site sync",
"statusPending": "Pending",
"statusRunning": "Running",
"statusDone": "Done",
+25
View File
@@ -645,6 +645,30 @@
"conflictSkip": "跳过已存在文件",
"conflictOverwrite": "覆盖已存在文件与数据库",
"syncTitle": "跨后端同步",
"siteSyncTitle": "两站点同步",
"siteSyncSubtitle": "将当前站点的媒体索引、图片、视频、文件、缩略图、HLS 分片和标签同步到另一台 PicHost。目标站点需已有同名用户。",
"siteSyncPushSubtitle": "将当前站点的媒体索引、图片、视频、文件、缩略图、HLS 分片和标签推送到另一台 PicHost。目标站点需已有同名用户。",
"siteSyncPullSubtitle": "从另一台 PicHost 拉取媒体索引、图片、视频、文件、缩略图、HLS 分片和标签。当前站点需已有同名用户。",
"siteSyncTarget": "目标站点地址",
"siteSyncSource": "来源站点地址",
"siteSyncToken": "目标站点管理员 API Token",
"siteSyncSourceToken": "来源站点管理员 API Token",
"siteSyncTokenPlaceholder": "输入目标站点管理员 Token",
"siteSyncSourceTokenPlaceholder": "输入来源站点管理员 Token",
"siteSyncStart": "开始同步",
"siteSyncStartPush": "开始推送",
"siteSyncStartPull": "开始拉取",
"siteSyncDirection": "同步策略:",
"siteSyncPush": "推送到目标站点",
"siteSyncPull": "从目标站点拉取",
"siteSyncConflict": "冲突策略:",
"siteSyncSkip": "跳过已存在",
"siteSyncOverwrite": "覆盖已存在",
"siteSyncResume": "继续上次同步",
"siteSyncScanProgress": "媒体扫描进度",
"siteSyncFileProgress": "已同步文件数量",
"siteSyncStarted": "两站点同步任务已开始",
"siteSyncFailed": "启动两站点同步失败",
"syncMigrateTitle": "跨后端迁移",
"syncMigrateSubtitle": "在已配置后端之间迁移文件",
"syncSubtitle": "在已配置的后端之间拷贝文件并更新索引",
@@ -694,6 +718,7 @@
"jobTypeExport": "站点导出",
"jobTypeRestore": "站点恢复",
"jobTypeSync": "跨后端同步",
"jobTypeSiteSync": "两站点同步",
"statusPending": "等待中",
"statusRunning": "进行中",
"statusDone": "已完成",
+59
View File
@@ -0,0 +1,59 @@
import { requireAdminAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { serializeBackupJob } from '../../utils/backup/api'
import {
resumeSiteSyncJob,
startSiteSyncJob
} from '../../utils/backup/runner'
interface SiteSyncBody {
targetBaseUrl?: string
token?: string
conflictMode?: 'skip' | 'overwrite'
direction?: 'push' | 'pull'
resumeJobId?: string
}
export default defineEventHandler(async (event) => {
const user = await requireAdminAuth(event)
const body = await readBody<SiteSyncBody>(event).catch(() => null)
const targetBaseUrl = body?.targetBaseUrl?.trim() ?? ''
const token = body?.token?.trim() ?? ''
const conflictMode = body?.conflictMode === 'skip' ? 'skip' : 'overwrite'
const direction = body?.direction === 'pull' ? 'pull' : 'push'
if (!token) {
createApiError(event, 'INVALID_REQUEST', '请输入目标站点管理员 API Token', 400)
}
try {
let job
if (body?.resumeJobId) {
job = resumeSiteSyncJob(body.resumeJobId, token, user.id)
} else {
let targetUrl: URL
try {
targetUrl = new URL(targetBaseUrl)
if (targetUrl.protocol !== 'http:' && targetUrl.protocol !== 'https:') {
throw new Error('invalid protocol')
}
} catch {
createApiError(event, 'INVALID_REQUEST', '请输入有效的目标站点地址', 400)
}
job = startSiteSyncJob({
targetBaseUrl: targetUrl.toString(),
conflictMode,
direction
}, token, user.id)
}
return { job: serializeBackupJob(job) }
} catch (error) {
if (error instanceof Error && error.message === 'BACKUP_JOB_RUNNING') {
createApiError(event, 'CONFLICT', '已有备份或迁移任务正在运行', 409)
}
if (error instanceof Error && error.message === 'SITE_SYNC_JOB_NOT_RESUMABLE') {
createApiError(event, 'INVALID_REQUEST', '该同步任务无法继续', 400)
}
throw error
}
})
+2 -2
View File
@@ -3,7 +3,7 @@ import { assertImageOwnership, requireApiOrAdminAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { logActivity } from '../../utils/activity-log'
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
import { validateFileKey, validateVideoKey } from '../../utils/image-key'
import { validateStorageKey, validateVideoKey } from '../../utils/image-key'
import { deleteImage, headImage } from '../../utils/storage'
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
import { MAX_DELETE_BATCH } from '../../utils/constants'
@@ -31,7 +31,7 @@ export default defineEventHandler(async (event) => {
const failed: BatchDeleteResponse['failed'] = []
for (const key of keys) {
if (!validateFileKey(key) && !validateVideoKey(key)) {
if (!validateStorageKey(key)) {
failed.push({ key, error: { code: 'INVALID_IMAGE_KEY', message: '无效的文件路径' } })
continue
}
+2 -2
View File
@@ -3,7 +3,7 @@ import { assertImageOwnership, requireApiOrAdminAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { logActivity } from '../../utils/activity-log'
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
import { validateFileKey, validateVideoKey } from '../../utils/image-key'
import { validateStorageKey, validateVideoKey } from '../../utils/image-key'
import { deleteImage, headImage } from '../../utils/storage'
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
import { logInfo } from '../../utils/logger'
@@ -20,7 +20,7 @@ export default defineEventHandler(async (event) => {
const body = await readBody<DeleteBody>(event).catch(() => null)
const key = (typeof query.key === 'string' ? query.key : undefined) ?? body?.key
if (!key || (!validateFileKey(key) && !validateVideoKey(key))) {
if (!key || !validateStorageKey(key)) {
createApiError(event, 'INVALID_IMAGE_KEY', '无效的文件路径', 400)
}
+38
View File
@@ -0,0 +1,38 @@
import { requireSiteSyncAuth } from '../../utils/access'
import { listAllImageIndexRows } from '../../utils/image-index'
import { listUserIdUsernameMap } from '../../utils/db'
import { collectMediaAssetKeys } from '../../utils/media-assets'
import { getBackendForKey } from '../../utils/storage/resolver'
import type { SiteSyncInventory } from '../../utils/site-sync/protocol'
export default defineEventHandler(async (event): Promise<SiteSyncInventory> => {
await requireSiteSyncAuth(event)
const rows = listAllImageIndexRows()
const objects = new Map<string, number>()
for (const row of rows) {
const backend = await getBackendForKey(row.key)
let keys: string[]
try {
keys = await collectMediaAssetKeys(row, backend)
} catch {
keys = [
...(Number(row.source_deleted ?? 0) !== 1 ? [row.key] : []),
...(row.thumb_key ? [row.thumb_key] : []),
...(row.hls_key ? [row.hls_key] : [])
]
}
for (const key of new Set(keys)) {
const stored = await backend.head(key)
if (stored) objects.set(key, stored.size)
}
}
return {
users: [...listUserIdUsernameMap().values()].sort(),
mediaKeys: rows.map(row => row.key),
objects: [...objects].map(([key, size]) => ({ key, size }))
}
})
+75
View File
@@ -0,0 +1,75 @@
import { getQuery } from 'h3'
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { listUserIdUsernameMap } from '../../utils/db'
import {
getImageIndexRow,
getImageTotals,
listImageIndexRowsAfter
} from '../../utils/image-index'
import { validateStorageKey } from '../../utils/image-key'
import { collectMediaAssetKeys } from '../../utils/media-assets'
import { getBackendForKey } from '../../utils/storage/resolver'
import { getTagsForImageKeys } from '../../utils/tags'
import { toSiteSyncMediaTransfer } from '../../utils/site-sync/manifest'
import type {
SiteSyncMediaAsset,
SiteSyncMediaBundle,
SiteSyncMediaPage
} from '../../utils/site-sync/protocol'
export default defineEventHandler(async (event): Promise<SiteSyncMediaPage> => {
await requireSiteSyncAuth(event)
const query = getQuery(event)
const key = typeof query.key === 'string' ? query.key.trim() : ''
const after = typeof query.after === 'string' ? query.after : ''
const limit = Math.min(
100,
Math.max(1, Number.parseInt(String(query.limit ?? '50'), 10) || 50)
)
if (key && !validateStorageKey(key)) {
createApiError(event, 'INVALID_REQUEST', '媒体 key 无效', 400)
}
const rows = key
? [getImageIndexRow(key)].filter(row => row !== null)
: listImageIndexRowsAfter(after, limit)
if (key && rows.length === 0) {
createApiError(event, 'IMAGE_NOT_FOUND', '目标媒体不存在', 404)
}
const ownerMap = listUserIdUsernameMap()
const tagMap = getTagsForImageKeys(rows.map(row => row.key))
const items: SiteSyncMediaBundle[] = []
for (const row of rows) {
const backend = await getBackendForKey(row.key)
const keys = await collectMediaAssetKeys(row, backend)
const assets: SiteSyncMediaAsset[] = []
for (const assetKey of new Set(keys)) {
const stored = await backend.head(assetKey)
if (!stored) {
throw new Error(`目标媒体对象不存在:${assetKey}`)
}
assets.push({ key: assetKey, size: stored.size })
}
items.push({
media: toSiteSyncMediaTransfer(
row,
row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
tagMap.get(row.key) ?? []
),
assets
})
}
return {
items,
totalMedia: getImageTotals().count,
nextCursor: key || rows.length < limit
? null
: rows.at(-1)?.key ?? null
}
})
+26
View File
@@ -0,0 +1,26 @@
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { validateStorageKey } from '../../utils/image-key'
import { applySiteSyncMedia } from '../../utils/site-sync/apply'
import type { SiteSyncMediaTransfer } from '../../utils/site-sync/protocol'
interface Body {
media?: SiteSyncMediaTransfer
conflictMode?: 'skip' | 'overwrite'
}
export default defineEventHandler(async (event) => {
const admin = await requireSiteSyncAuth(event)
const body = await readBody<Body>(event).catch(() => null)
const media = body?.media
if (!media || !validateStorageKey(media.key)) {
createApiError(event, 'INVALID_REQUEST', '媒体同步参数无效', 400)
}
const conflictMode = body?.conflictMode === 'skip' ? 'skip' : 'overwrite'
try {
return await applySiteSyncMedia(media, conflictMode, admin.id)
} catch (error) {
const message = error instanceof Error ? error.message : '媒体同步失败'
createApiError(event, 'INVALID_REQUEST', message, 400)
}
})
+26
View File
@@ -0,0 +1,26 @@
import { getQuery, sendStream, setResponseHeader } from 'h3'
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { validateStorageKey } from '../../utils/image-key'
import { getBackendForKey } from '../../utils/storage/resolver'
export default defineEventHandler(async (event) => {
await requireSiteSyncAuth(event)
const query = getQuery(event)
const key = typeof query.key === 'string' ? query.key.trim() : ''
if (!validateStorageKey(key)) {
createApiError(event, 'INVALID_REQUEST', '同步对象 key 无效', 400)
}
const backend = await getBackendForKey(key)
const stored = await backend.head(key)
if (!stored) {
createApiError(event, 'IMAGE_NOT_FOUND', `目标对象不存在:${key}`, 404)
}
setResponseHeader(event, 'Content-Type', stored.contentType)
setResponseHeader(event, 'Content-Length', stored.size)
setResponseHeader(event, 'Cache-Control', 'no-store')
return sendStream(event, await backend.createStream(key))
})
+54
View File
@@ -0,0 +1,54 @@
import { readMultipartFormData } from 'h3'
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { MAX_VIDEO_FILE_SIZE } from '../../utils/constants'
import { findUserByUsername } from '../../utils/db'
import { validateStorageKey } from '../../utils/image-key'
import { getActiveBackend } from '../../utils/storage/resolver'
function field(form: Awaited<ReturnType<typeof readMultipartFormData>>, name: string): string {
const part = form?.find(item => item.name === name)
return part?.data?.length ? new TextDecoder().decode(part.data).trim() : ''
}
export default defineEventHandler(async (event) => {
await requireSiteSyncAuth(event)
const form = await readMultipartFormData(event)
if (!form?.length) createApiError(event, 'INVALID_REQUEST', '未收到同步数据', 400)
const key = field(form, 'key')
const file = form.find(part => part.name === 'file' && part.data?.length)
if (!validateStorageKey(key) || !file?.data) {
createApiError(event, 'INVALID_REQUEST', '同步文件参数无效', 400)
}
if (file.data.length > MAX_VIDEO_FILE_SIZE) {
createApiError(event, 'FILE_TOO_LARGE', '同步文件超过 500 MB', 413)
}
const conflictMode = field(form, 'conflictMode') === 'skip' ? 'skip' : 'overwrite'
const backend = await getActiveBackend()
if (conflictMode === 'skip' && await backend.head(key)) {
return { success: true, skipped: true }
}
const ownerUsername = field(form, 'ownerUsername')
const owner = ownerUsername ? findUserByUsername(ownerUsername) : null
if (ownerUsername && !owner) {
createApiError(event, 'INVALID_REQUEST', `目标站点缺少用户:${ownerUsername}`, 400)
}
await backend.put(key, new Uint8Array(file.data), {
originalName: field(form, 'originalName') || file.filename || key.split('/').pop() || 'file',
uploadedAt: field(form, 'uploadedAt') || new Date().toISOString(),
contentType: file.type || field(form, 'contentType') || 'application/octet-stream',
size: file.data.length,
userId: owner?.id ?? null,
mediaType: key.startsWith('videos/')
? 'video'
: key.startsWith('files/')
? 'file'
: 'image'
})
return { success: true, skipped: false }
})
+10
View File
@@ -0,0 +1,10 @@
import { requireSiteSyncAuth } from '../../utils/access'
import { listUserIdUsernameMap } from '../../utils/db'
import type { SiteSyncUsers } from '../../utils/site-sync/protocol'
export default defineEventHandler(async (event): Promise<SiteSyncUsers> => {
await requireSiteSyncAuth(event)
return {
users: [...listUserIdUsernameMap().values()].sort()
}
})
+31 -1
View File
@@ -1,5 +1,5 @@
import type { H3Event } from 'h3'
import { findUserByApiToken, getPrimaryAdminUserId } from './db'
import { findUserById, findUserByApiToken, getPrimaryAdminUserId } from './db'
import { getAdminSecret, getApiUploadToken } from './env'
import { createApiError } from './api-error'
import {
@@ -110,6 +110,36 @@ export function resolveUserIdFromApiToken(
return null
}
/** 两站点同步:全站 Token 或管理员个人 Token;也允许管理员网页会话。 */
export async function requireSiteSyncAuth(event: H3Event): Promise<AuthUser> {
if (devBypass(event)) {
return { id: 0, username: 'dev', role: 'admin' }
}
if (verifyGlobalApiUploadToken(event)) {
const adminId = getPrimaryAdminUserId()
if (adminId != null) {
const admin = findUserById(adminId)
if (admin) {
return { id: admin.id, username: admin.username, role: admin.role }
}
}
}
const userId = resolveUserIdFromApiToken(event)
if (userId != null) {
const user = findUserById(userId)
if (user?.role === 'admin') {
return { id: user.id, username: user.username, role: user.role }
}
}
const sessionUser = await getCurrentUser(event)
if (sessionUser?.role === 'admin') return sessionUser
createApiError(event, 'FORBIDDEN', '需要目标站点管理员同步 Token', 403)
}
export function verifyApiUploadToken(event: H3Event): boolean {
return verifyGlobalApiUploadToken(event)
|| resolveUserIdFromApiToken(event) !== null
+2
View File
@@ -9,6 +9,8 @@ export {
startFullExportJob,
startFullRestoreJob,
startBackendSyncJob,
startSiteSyncJob,
resumeSiteSyncJob,
isBackupRunnerBusy,
cancelBackupJob
} from './runner'
+1 -1
View File
@@ -2,7 +2,7 @@ import { randomBytes } from 'node:crypto'
import { getDb, getPrimaryAdminUserId } from '../db'
import { ensureBackupSchema } from './schema'
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync'
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
export interface BackupJob {
+53
View File
@@ -1,6 +1,7 @@
import { runBackendSyncJob } from './backend-sync'
import { runFullExportJob } from './full-export'
import { runFullRestoreJob } from './full-restore'
import { registerSiteSyncToken, runSiteSyncJob, type SiteSyncOptions } from '../site-sync/runner'
import {
createBackupJob,
getBackupJob,
@@ -114,3 +115,55 @@ export function startBackendSyncJob(options: {
scheduleBackupJob(job, () => runBackendSyncJob(job.id, options))
return getBackupJob(job.id)!
}
export function startSiteSyncJob(
options: SiteSyncOptions,
token: string,
userId?: number | null
): BackupJob {
claimJobSlot()
const job = createBackupJob({
type: 'site_sync',
userId,
message: '等待开始…',
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: options.direction
}
})
registerSiteSyncToken(job.id, token)
scheduleBackupJob(job, () => runSiteSyncJob(job.id, options))
return getBackupJob(job.id)!
}
export function resumeSiteSyncJob(
jobId: string,
token: string,
userId?: number | null
): BackupJob {
claimJobSlot()
const job = getBackupJob(jobId, userId ?? undefined)
if (
!job
|| job.type !== 'site_sync'
|| job.status !== 'failed'
|| job.payload.resumable !== true
) {
throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
}
const targetBaseUrl = typeof job.payload.targetBaseUrl === 'string'
? job.payload.targetBaseUrl
: ''
const conflictMode = job.payload.conflictMode === 'skip' ? 'skip' : 'overwrite'
const direction = job.payload.direction === 'pull' ? 'pull' : 'push'
if (!targetBaseUrl) throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
registerSiteSyncToken(job.id, token)
scheduleBackupJob(job, () => runSiteSyncJob(job.id, {
targetBaseUrl,
conflictMode,
direction
}))
return getBackupJob(job.id)!
}
+41 -1
View File
@@ -1,11 +1,50 @@
import { getDb } from '../db'
function backupJobsTableSql(): string {
const row = getDb().prepare(`
SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'backup_jobs'
`).get() as { sql?: string } | undefined
return row?.sql ?? ''
}
function migrateBackupJobTypes(): void {
if (backupJobsTableSql().includes('\'site_sync\'')) return
const db = getDb()
db.exec(`
CREATE TABLE backup_jobs_new (
id TEXT PRIMARY KEY,
user_id INTEGER,
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
progress INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0,
message TEXT NOT NULL DEFAULT '',
payload_json TEXT NOT NULL DEFAULT '{}',
output_path TEXT,
error TEXT,
created_at TEXT NOT NULL,
finished_at TEXT
);
INSERT INTO backup_jobs_new
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
SELECT id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at
FROM backup_jobs;
DROP TABLE backup_jobs;
ALTER TABLE backup_jobs_new RENAME TO backup_jobs;
CREATE INDEX IF NOT EXISTS idx_backup_jobs_created_at
ON backup_jobs(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_backup_jobs_user_created
ON backup_jobs(user_id, created_at DESC);
`)
}
export function ensureBackupSchema(): void {
getDb().exec(`
CREATE TABLE IF NOT EXISTS backup_jobs (
id TEXT PRIMARY KEY,
user_id INTEGER,
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync')),
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
progress INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0,
@@ -36,4 +75,5 @@ export function ensureBackupSchema(): void {
WHERE user_id IS NULL
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
`)
migrateBackupJobTypes()
}
+23 -7
View File
@@ -35,6 +35,7 @@ export interface ImageIndexRow {
backend_id: string
user_id: number | null
folder: string
media_type: MediaType
original_name: string
content_type: string
size: number
@@ -71,6 +72,7 @@ function rowToStoredImage(row: ImageIndexRow): StoredImage {
contentType: row.content_type,
size: row.size,
userId: row.user_id,
mediaType: row.media_type,
hlsKey: row.hls_key ?? null,
transcodeStatus: (row.transcode_status ?? 'none') as TranscodeStatus,
transcodeProgress: Number(row.transcode_progress ?? 0),
@@ -232,7 +234,7 @@ export async function purgeOrphanImageIndexRows(): Promise<number> {
export function getImageIndexRow(key: string): ImageIndexRow | null {
ensureStorageSchema()
const row = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE key = ?
`).get(key) as ImageIndexRow | undefined
@@ -425,16 +427,30 @@ export function getImageTotals(): { count: number, bytes: number } {
export function listAllImageIndexRows(): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
ORDER BY key ASC
`).all() as unknown as ImageIndexRow[]
}
export function listImageIndexRowsAfter(
afterKey: string,
limit: number
): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE key > ?
ORDER BY key ASC
LIMIT ?
`).all(afterKey, limit) as unknown as ImageIndexRow[]
}
export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE user_id = ?
ORDER BY key ASC
@@ -444,7 +460,7 @@ export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
export function listImageIndexRowsByBackend(backendId: string): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE backend_id = ?
ORDER BY key ASC
@@ -687,7 +703,7 @@ export async function listImages(options: {
const backendSql = buildBackendFilterSql(options.backendId, params)
const tagSql = buildTagFilterSql(options.tagFilter, params)
const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
ORDER BY uploaded_at DESC, key DESC
@@ -733,7 +749,7 @@ export async function searchImages(options: {
const backendSql = buildBackendFilterSql(options.backendId, params)
const tagSql = buildTagFilterSql(options.tagFilter, params)
const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
ORDER BY uploaded_at DESC, key DESC
@@ -867,7 +883,7 @@ export async function getUserScopedStorageStats(
export async function listImageKeysForAutoDelete(): Promise<StoredImage[]> {
ensureStorageSchema()
const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE media_type = 'image'
ORDER BY uploaded_at ASC
+1
View File
@@ -8,6 +8,7 @@ const row: ImageIndexRow = {
backend_id: 'local',
user_id: 1,
folder: 'videos',
media_type: 'video',
original_name: 'example.mp4',
content_type: 'video/mp4',
size: 100,
+79
View File
@@ -0,0 +1,79 @@
import { findUserByUsername } from '../db'
import {
getImageIndexRow,
insertImageIndex,
updateImageThumb,
updateImageTranscode
} from '../image-index'
import { getActiveBackend } from '../storage/resolver'
import {
createTag,
findTagByName,
setImageTags
} from '../tags'
import type { SiteSyncMediaTransfer } from './protocol'
export async function applySiteSyncMedia(
media: SiteSyncMediaTransfer,
conflictMode: 'skip' | 'overwrite',
actorUserId: number
): Promise<{ success: true, skipped: boolean }> {
if (conflictMode === 'skip' && getImageIndexRow(media.key)) {
return { success: true, skipped: true }
}
const owner = media.ownerUsername
? findUserByUsername(media.ownerUsername)
: null
if (media.ownerUsername && !owner) {
throw new Error(`当前站点缺少用户:${media.ownerUsername}`)
}
const backend = await getActiveBackend()
const requiredKeys = [
...(!media.sourceDeleted ? [media.key] : []),
...(media.thumbKey ? [media.thumbKey] : []),
...(media.hlsKey ? [media.hlsKey] : [])
]
for (const key of requiredKeys) {
if (!await backend.head(key)) {
throw new Error(`同步对象尚未写入:${key}`)
}
}
insertImageIndex({
key: media.key,
backendId: backend.id,
userId: owner?.id ?? null,
originalName: media.originalName,
contentType: media.contentType,
size: media.size,
uploadedAt: media.uploadedAt,
mediaType: media.mediaType
})
if (media.thumbKey) updateImageThumb(media.key, media.thumbKey)
if (media.mediaType === 'video') {
updateImageTranscode(media.key, {
status: media.transcodeStatus,
hlsKey: media.hlsKey,
progress: media.transcodeProgress,
preset: media.transcodePreset,
sourceDeleted: media.sourceDeleted,
error: media.transcodeError
})
}
const tagOwnerId = owner?.id ?? actorUserId
const tagIds: number[] = []
for (const tag of media.tags) {
const existing = findTagByName(tagOwnerId, tag.name)
const item = existing
? { id: existing.id }
: createTag(tagOwnerId, tag.name, tag.color)
tagIds.push(item.id)
}
setImageTags(media.key, tagIds, tagOwnerId, true)
return { success: true, skipped: false }
}
+38
View File
@@ -0,0 +1,38 @@
import type { ImageIndexRow } from '../image-index'
import { getTagsForImageKeys } from '../tags'
import type {
SiteSyncMediaTransfer,
SiteSyncTag
} from './protocol'
export function toSiteSyncMediaTransfer(
row: ImageIndexRow,
ownerUsername: string | null,
tags: SiteSyncTag[]
): SiteSyncMediaTransfer {
return {
key: row.key,
ownerUsername,
originalName: row.original_name,
contentType: row.content_type,
size: row.size,
uploadedAt: row.uploaded_at,
mediaType: row.media_type,
thumbKey: row.thumb_key,
hlsKey: row.hls_key,
transcodeStatus: row.transcode_status as SiteSyncMediaTransfer['transcodeStatus'],
transcodeProgress: Number(row.transcode_progress ?? 0),
transcodePreset: row.transcode_preset,
transcodeError: row.transcode_error,
sourceDeleted: Number(row.source_deleted ?? 0) === 1,
tags
}
}
export function toSiteSyncMediaTransferWithTags(
row: ImageIndexRow,
ownerUsername: string | null
): SiteSyncMediaTransfer {
const tags = getTagsForImageKeys([row.key]).get(row.key) ?? []
return toSiteSyncMediaTransfer(row, ownerUsername, tags)
}
+59
View File
@@ -0,0 +1,59 @@
import type { MediaType, TranscodeStatus } from '../storage/types'
export interface SiteSyncTag {
name: string
color: string
}
export interface SiteSyncMediaTransfer {
key: string
ownerUsername: string | null
originalName: string
contentType: string
size: number
uploadedAt: string
mediaType: MediaType
thumbKey: string | null
hlsKey: string | null
transcodeStatus: TranscodeStatus
transcodeProgress: number
transcodePreset: string | null
transcodeError: string | null
sourceDeleted: boolean
tags: SiteSyncTag[]
}
export interface SiteSyncInventory {
users: string[]
mediaKeys: string[]
objects: Array<{ key: string, size: number }>
}
export interface SiteSyncMediaAsset {
key: string
size: number
}
export interface SiteSyncMediaBundle {
media: SiteSyncMediaTransfer
assets: SiteSyncMediaAsset[]
}
export interface SiteSyncMediaPage {
items: SiteSyncMediaBundle[]
totalMedia: number
nextCursor: string | null
}
export interface SiteSyncUsers {
users: string[]
}
export type SiteSyncDirection = 'push' | 'pull'
export interface SiteSyncObjectOptions {
targetBaseUrl: string
token: string
conflictMode: 'skip' | 'overwrite'
direction?: SiteSyncDirection
}
+55
View File
@@ -0,0 +1,55 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { fetchSiteSyncJson, retrySiteSyncOperation } from './runner'
afterEach(() => {
vi.unstubAllGlobals()
})
describe('fetchSiteSyncJson', () => {
it('retries a broken connection up to three attempts', async () => {
const fetchMock = vi.fn()
.mockRejectedValueOnce(new Error('connection reset'))
.mockRejectedValueOnce(new Error('connection reset'))
.mockResolvedValueOnce(new Response('{"ok":true}', {
status: 200,
headers: { 'Content-Type': 'application/json' }
}))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).resolves.toEqual({ ok: true })
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('retries client errors because every sync failure gets three attempts', async () => {
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
new Response('{"message":"invalid"}', {
status: 400,
headers: { 'Content-Type': 'application/json' }
})
))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('invalid')
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('fails after three server errors', async () => {
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
new Response('{"message":"unavailable"}', {
status: 503,
headers: { 'Content-Type': 'application/json' }
})
))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('unavailable')
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('retries local sync failures and stops after the third attempt', async () => {
const operation = vi.fn().mockRejectedValue(new Error('database is busy'))
await expect(retrySiteSyncOperation(operation)).rejects.toThrow('database is busy')
expect(operation).toHaveBeenCalledTimes(3)
})
})
+641
View File
@@ -0,0 +1,641 @@
import { basename } from 'node:path'
import { contentTypeFromKey } from '../content-type'
import {
getImageIndexRow,
listAllImageIndexRows,
type ImageIndexRow
} from '../image-index'
import {
findUserByUsername,
getPrimaryAdminUserId,
listUserIdUsernameMap
} from '../db'
import { collectMediaAssetKeys } from '../media-assets'
import {
getActiveBackend,
getBackendForKey
} from '../storage/resolver'
import type { StorageBackend } from '../storage/types'
import {
getBackupJob,
markJobDone,
markJobFailed,
updateBackupJob
} from '../backup/jobs'
import { applySiteSyncMedia } from './apply'
import { toSiteSyncMediaTransferWithTags } from './manifest'
import type {
SiteSyncDirection,
SiteSyncInventory,
SiteSyncMediaBundle,
SiteSyncMediaPage,
SiteSyncUsers
} from './protocol'
export interface SiteSyncOptions {
targetBaseUrl: string
conflictMode: 'skip' | 'overwrite'
direction: SiteSyncDirection
}
const tokens = (globalThis as typeof globalThis & {
__pichostSiteSyncTokens?: Map<string, string>
}).__pichostSiteSyncTokens ?? new Map<string, string>()
;(globalThis as typeof globalThis & {
__pichostSiteSyncTokens?: Map<string, string>
}).__pichostSiteSyncTokens = tokens
export function registerSiteSyncToken(jobId: string, token: string): void {
tokens.set(jobId, token)
}
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = []
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
}
return Buffer.concat(chunks)
}
function targetUrl(baseUrl: string, path: string): string {
return `${baseUrl.replace(/\/+$/, '')}${path}`
}
async function responseError(response: Response): Promise<Error> {
const payload = await response.json().catch(() => null) as {
message?: string
statusMessage?: string
} | null
return new Error(payload?.message || payload?.statusMessage || `目标站点返回 ${response.status}`)
}
const SITE_SYNC_MAX_ATTEMPTS = 3
export async function retrySiteSyncOperation<T>(
operation: () => Promise<T>
): Promise<T> {
let lastError: Error | null = null
for (let attempt = 1; attempt <= SITE_SYNC_MAX_ATTEMPTS; attempt++) {
try {
return await operation()
} catch (error) {
lastError = error instanceof Error ? error : new Error('两站点同步失败')
if (attempt < SITE_SYNC_MAX_ATTEMPTS) {
await new Promise(resolve => setTimeout(resolve, attempt * 500))
}
}
}
throw lastError ?? new Error('两站点同步失败')
}
async function fetchSiteSyncJsonOnce<T>(
url: string,
token: string,
init?: RequestInit
): Promise<T> {
const response = await fetch(url, {
...init,
headers: {
'Auth-Token': token,
...(init?.headers ?? {})
}
})
if (!response.ok) throw await responseError(response)
return await response.json() as T
}
export async function fetchSiteSyncJson<T>(
url: string,
token: string,
init?: RequestInit
): Promise<T> {
return retrySiteSyncOperation(() => (
fetchSiteSyncJsonOnce<T>(url, token, init)
))
}
async function uploadObject(
options: SiteSyncOptions,
token: string,
backend: StorageBackend,
row: ImageIndexRow,
ownerUsername: string | null,
key: string
): Promise<void> {
const stored = await backend.head(key)
if (!stored) throw new Error(`源文件不存在:${key}`)
const bytes = await streamToBuffer(await backend.createStream(key))
const form = new FormData()
const isPrimary = key === row.key
form.append('key', key)
form.append('conflictMode', options.conflictMode)
form.append('ownerUsername', ownerUsername ?? '')
form.append('originalName', isPrimary ? row.original_name : basename(key))
form.append('uploadedAt', row.uploaded_at)
const contentType = isPrimary ? row.content_type : contentTypeFromKey(key)
form.append('contentType', contentType)
form.append(
'file',
new Blob([new Uint8Array(bytes)], { type: contentType }),
basename(key)
)
await fetchSiteSyncJsonOnce<{ success: boolean }>(
targetUrl(options.targetBaseUrl, '/api/site-sync/object'),
token,
{ method: 'POST', body: form }
)
}
async function downloadSiteSyncObject(
options: SiteSyncOptions,
token: string,
key: string
): Promise<Buffer> {
const response = await fetch(
targetUrl(
options.targetBaseUrl,
`/api/site-sync/object?key=${encodeURIComponent(key)}`
),
{
headers: {
'Auth-Token': token
}
}
)
if (!response.ok) throw await responseError(response)
return Buffer.from(await response.arrayBuffer())
}
async function runSiteSyncPull(
jobId: string,
options: SiteSyncOptions,
token: string
): Promise<void> {
const job = getBackupJob(jobId)
const checkpoint = job?.payload ?? {}
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
? checkpoint.lastMediaKey
: ''
let scanned = Number(checkpoint.scanned ?? 0)
let added = Number(checkpoint.added ?? 0)
let skipped = Number(checkpoint.skipped ?? 0)
let filesSynced = Number(checkpoint.filesSynced ?? 0)
let totalFiles = Number(checkpoint.totalFiles ?? 0)
let totalMedia = Number(checkpoint.totalMedia ?? job?.total ?? 0)
const failedKeys = new Set(
Array.isArray(checkpoint.failedKeys)
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
: []
)
const saveCheckpoint = (message: string) => {
updateBackupJob(jobId, {
progress: scanned,
total: totalMedia,
message,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'pull',
lastMediaKey,
scanned,
added,
skipped,
failed: failedKeys.size,
filesSynced,
totalFiles,
totalMedia,
failedKeys: [...failedKeys],
resumable: true
}
})
}
const progressMessage = (suffix = '') => (
`扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}${suffix}`
)
const failAndCheckpoint = (error: unknown) => {
const message = progressMessage(' · 已保存断点')
saveCheckpoint(message)
markJobFailed(
jobId,
error instanceof Error ? error.message : '从目标站点拉取失败',
message
)
}
try {
const targetUsers = new Set(
(await fetchSiteSyncJson<SiteSyncUsers>(
targetUrl(options.targetBaseUrl, '/api/site-sync/users'),
token
)).users
)
const localUserMap = listUserIdUsernameMap()
const localUsers = new Set(localUserMap.values())
const missingUsers = [...targetUsers].filter(name => !localUsers.has(name))
if (missingUsers.length) {
throw new Error(`当前站点缺少同名用户:${missingUsers.join('、')}`)
}
const backend = await retrySiteSyncOperation(() => getActiveBackend())
const actorUserId = job?.userId ?? getPrimaryAdminUserId() ?? 0
const readMediaPage = (path: string) => fetchSiteSyncJson<SiteSyncMediaPage>(
targetUrl(options.targetBaseUrl, path),
token
)
const processBundleOnce = async (
bundle: SiteSyncMediaBundle,
isRetry: boolean
): Promise<'added' | 'skipped'> => {
const { media, assets } = bundle
if (
!isRetry
&& options.conflictMode === 'skip'
&& getImageIndexRow(media.key)
) {
return 'skipped'
}
const owner = media.ownerUsername
? findUserByUsername(media.ownerUsername)
: null
for (const asset of assets) {
if (
options.conflictMode === 'skip'
&& await backend.head(asset.key)
) {
continue
}
const bytes = await downloadSiteSyncObject(
options,
token,
asset.key
)
if (bytes.length !== asset.size) {
throw new Error(
`同步对象大小不一致:${asset.key},预期 ${asset.size},实际 ${bytes.length}`
)
}
const isPrimary = asset.key === media.key
await backend.put(asset.key, new Uint8Array(bytes), {
originalName: isPrimary ? media.originalName : basename(asset.key),
uploadedAt: media.uploadedAt,
contentType: isPrimary
? media.contentType
: contentTypeFromKey(asset.key),
size: bytes.length,
userId: owner?.id ?? null,
mediaType: isPrimary
? media.mediaType
: asset.key.startsWith('videos/')
? 'video'
: asset.key.startsWith('files/')
? 'file'
: 'image'
})
}
await applySiteSyncMedia(media, 'overwrite', actorUserId)
return 'added'
}
const processBundle = (bundle: SiteSyncMediaBundle, isRetry: boolean) => (
retrySiteSyncOperation(() => processBundleOnce(bundle, isRetry))
)
const recordOutcome = (outcome: 'added' | 'skipped', fileCount: number) => {
if (outcome === 'added') added += 1
else skipped += 1
filesSynced += fileCount
totalFiles = Math.max(totalFiles, filesSynced)
}
for (const key of [...failedKeys]) {
try {
const page = await readMediaPage(
`/api/site-sync/media?key=${encodeURIComponent(key)}`
)
const bundle = page.items[0]
if (!bundle) throw new Error(`目标媒体不存在:${key}`)
const outcome = await processBundle(bundle, true)
recordOutcome(outcome, bundle.assets.length)
failedKeys.delete(key)
if (key > lastMediaKey) lastMediaKey = key
saveCheckpoint(progressMessage())
} catch (error) {
failedKeys.add(key)
failAndCheckpoint(error)
return
}
}
let cursor = lastMediaKey
let lastCheckpointAt = 0
while (true) {
let page: SiteSyncMediaPage
try {
page = await readMediaPage(
`/api/site-sync/media?limit=50${cursor ? `&after=${encodeURIComponent(cursor)}` : ''}`
)
} catch (error) {
failAndCheckpoint(error)
return
}
totalMedia = Math.max(totalMedia, page.totalMedia)
for (const bundle of page.items) {
scanned += 1
try {
const outcome = await processBundle(bundle, false)
recordOutcome(outcome, bundle.assets.length)
failedKeys.delete(bundle.media.key)
} catch (error) {
failedKeys.add(bundle.media.key)
lastMediaKey = bundle.media.key
failAndCheckpoint(error)
return
}
lastMediaKey = bundle.media.key
const now = Date.now()
if (
now - lastCheckpointAt >= 1000
|| scanned === totalMedia
|| !page.nextCursor
) {
saveCheckpoint(progressMessage())
lastCheckpointAt = now
}
}
if (!page.nextCursor) break
cursor = page.nextCursor
}
if (failedKeys.size) {
markJobFailed(
jobId,
`${failedKeys.size} 条媒体拉取失败`,
progressMessage()
)
saveCheckpoint(progressMessage())
return
}
totalFiles = filesSynced
markJobDone(jobId, {
message: `扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'pull',
lastMediaKey,
scanned,
added,
skipped,
failed: 0,
filesSynced,
totalFiles,
totalMedia: totalMedia || scanned,
failedKeys: [],
resumable: false
}
})
} catch (error) {
failAndCheckpoint(error)
}
}
export async function runSiteSyncJob(
jobId: string,
options: SiteSyncOptions
): Promise<void> {
const token = tokens.get(jobId)
if (!token) throw new Error('同步 Token 不可用,请重新发起任务')
try {
if (options.direction === 'pull') {
await runSiteSyncPull(jobId, options, token)
return
}
const inventory = await fetchSiteSyncJson<SiteSyncInventory>(
targetUrl(options.targetBaseUrl, '/api/site-sync/inventory'),
token
)
const targetUsers = new Set(inventory.users)
const targetMedia = new Set(inventory.mediaKeys)
const targetObjects = new Map(inventory.objects.map(item => [item.key, item.size]))
const rows = listAllImageIndexRows()
const ownerMap = listUserIdUsernameMap()
const job = getBackupJob(jobId)
const checkpoint = job?.payload ?? {}
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
? checkpoint.lastMediaKey
: ''
let scanned = Number(checkpoint.scanned ?? 0)
let added = Number(checkpoint.added ?? 0)
let skipped = Number(checkpoint.skipped ?? 0)
let filesSynced = Number(checkpoint.filesSynced ?? 0)
const failedKeys = new Set(
Array.isArray(checkpoint.failedKeys)
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
: []
)
const missingUsers = [...new Set(
rows
.map(row => row.user_id == null ? null : ownerMap.get(row.user_id))
.filter((name): name is string => Boolean(name))
)].filter(name => !targetUsers.has(name))
if (missingUsers.length) {
throw new Error(`目标站点缺少同名用户:${missingUsers.join('、')}`)
}
const plans: Array<{
row: ImageIndexRow
ownerUsername: string | null
backend: StorageBackend
keys: string[]
}> = []
let totalFiles = 0
updateBackupJob(jobId, {
total: rows.length,
progress: scanned,
message: '正在统计同步文件…'
})
for (const row of rows) {
const backend = await getBackendForKey(row.key)
let assetKeys: string[]
try {
assetKeys = await collectMediaAssetKeys(row, backend)
} catch {
assetKeys = [
...(!Number(row.source_deleted ?? 0) ? [row.key] : []),
...(row.thumb_key ? [row.thumb_key] : []),
...(row.hls_key ? [row.hls_key] : [])
]
}
const keys = [...new Set(assetKeys)]
totalFiles += keys.length
plans.push({
row,
ownerUsername: row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
backend,
keys
})
}
const saveCheckpoint = (message: string) => {
updateBackupJob(jobId, {
progress: scanned,
total: rows.length,
message,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'push',
lastMediaKey,
scanned,
added,
skipped,
failed: failedKeys.size,
filesSynced,
totalFiles,
failedKeys: [...failedKeys],
resumable: true
}
})
}
plans.sort((a, b) => (
a.row.key < b.row.key
? -1
: a.row.key > b.row.key
? 1
: 0
))
lastMediaKey = lastMediaKey || ''
let lastCheckpointAt = 0
const processPlanOnce = async (plan: typeof plans[number], isRetry: boolean) => {
const { row, ownerUsername, backend, keys } = plan
if (
!isRetry
&& options.conflictMode === 'skip'
&& targetMedia.has(row.key)
) {
skipped += 1
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
failedKeys.delete(row.key)
return
}
for (const key of keys) {
const stored = await backend.head(key)
if (!stored) throw new Error(`源文件不存在:${key}`)
if (
options.conflictMode === 'skip'
&& targetObjects.get(key) === stored.size
) {
continue
}
await uploadObject(options, token, backend, row, ownerUsername, key)
}
await fetchSiteSyncJsonOnce<{ success: boolean }>(
targetUrl(options.targetBaseUrl, '/api/site-sync/media'),
token,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
media: toSiteSyncMediaTransferWithTags(row, ownerUsername),
conflictMode: options.conflictMode
})
}
)
failedKeys.delete(row.key)
added += 1
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
}
const processPlan = (
plan: typeof plans[number],
isRetry: boolean
) => retrySiteSyncOperation(() => processPlanOnce(plan, isRetry))
const retryPlans = plans.filter(plan => failedKeys.has(plan.row.key))
for (const plan of retryPlans) {
try {
await processPlan(plan, true)
lastMediaKey = plan.row.key
} catch {
failedKeys.add(plan.row.key)
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
saveCheckpoint(message)
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
return
}
}
for (const plan of plans) {
if (plan.row.key <= lastMediaKey) continue
scanned += 1
try {
await processPlan(plan, false)
} catch {
failedKeys.add(plan.row.key)
lastMediaKey = plan.row.key
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
saveCheckpoint(message)
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
return
}
lastMediaKey = plan.row.key
const now = Date.now()
if (now - lastCheckpointAt >= 1000 || scanned === rows.length || failedKeys.size) {
saveCheckpoint(
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
lastCheckpointAt = now
}
}
if (failedKeys.size) {
markJobFailed(
jobId,
`${failedKeys.size} 条媒体同步失败`,
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
saveCheckpoint(
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
return
}
markJobDone(jobId, {
message: `扫描 ${rows.length}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'push',
lastMediaKey,
scanned: rows.length,
added,
skipped,
failed: 0,
filesSynced: totalFiles,
totalFiles,
failedKeys: [],
resumable: false
}
})
} catch (error) {
const message = error instanceof Error ? error.message : '两站点同步失败'
markJobFailed(jobId, message)
throw error
} finally {
tokens.delete(jobId)
}
}