站点同步

This commit is contained in:
tianmo committed 2026-10-17 14:41:32 +08:00
1 parent a64be50b73
commit 9cb8552a99
28 files changed
+1683 -18

No files matched your search

+2 -1
View File
@@ -26,7 +26,8 @@ function typeLabel(type: BackupJobItem['type']): string {
const map: Record<BackupJobItem['type'], string> = { const map: Record<BackupJobItem['type'], string> = {
full_export: t('storage.backup.jobTypeExport'), full_export: t('storage.backup.jobTypeExport'),
full_restore: t('storage.backup.jobTypeRestore'), full_restore: t('storage.backup.jobTypeRestore'),
backend_sync: t('storage.backup.jobTypeSync') backend_sync: t('storage.backup.jobTypeSync'),
site_sync: t('storage.backup.jobTypeSiteSync')
} }
return map[type] return map[type]
} }
+256
View File
@@ -0,0 +1,256 @@
<script setup lang="ts">
import type {
SiteSyncDirection,
SiteSyncRequest
} from '~/types/backup'
const props = defineProps<{
busy: boolean
disabled?: boolean
activeJob?: {
progress: number
total: number
message: string
payload: Record<string, unknown>
} | null
resumableJob?: {
id: string
payload: Record<string, unknown>
} | null
}>()
const emit = defineEmits<{
sync: [body: SiteSyncRequest]
}>()
const { t } = useI18n()
const targetBaseUrl = ref('')
const token = ref('')
const conflictMode = ref<'skip' | 'overwrite'>('skip')
const direction = ref<SiteSyncDirection>('push')
watch(() => props.resumableJob, (job) => {
if (!job) return
direction.value = job.payload.direction === 'pull' ? 'pull' : 'push'
if (typeof job.payload.targetBaseUrl === 'string') {
targetBaseUrl.value = job.payload.targetBaseUrl
}
}, { immediate: true })
const canSubmit = computed(() =>
!props.busy
&& !props.disabled
&& Boolean(targetBaseUrl.value.trim())
&& Boolean(token.value.trim())
)
const canResume = computed(() =>
!props.busy
&& !props.disabled
&& Boolean(props.resumableJob)
&& Boolean(token.value.trim())
)
const fileProgress = computed(() => {
const payload = props.activeJob?.payload
return {
synced: Number(payload?.filesSynced ?? 0),
total: Number(payload?.totalFiles ?? 0)
}
})
const isPull = computed(() => direction.value === 'pull')
const directionSubtitle = computed(() => (
isPull.value
? t('storage.backup.siteSyncPullSubtitle')
: t('storage.backup.siteSyncPushSubtitle')
))
const endpointLabel = computed(() => (
isPull.value
? t('storage.backup.siteSyncSource')
: t('storage.backup.siteSyncTarget')
))
const tokenLabel = computed(() => (
isPull.value
? t('storage.backup.siteSyncSourceToken')
: t('storage.backup.siteSyncToken')
))
const tokenPlaceholder = computed(() => (
isPull.value
? t('storage.backup.siteSyncSourceTokenPlaceholder')
: t('storage.backup.siteSyncTokenPlaceholder')
))
const startLabel = computed(() => (
isPull.value
? t('storage.backup.siteSyncStartPull')
: t('storage.backup.siteSyncStartPush')
))
function submit(resume = false) {
if (resume ? !canResume.value : !canSubmit.value) return
emit('sync', {
targetBaseUrl: targetBaseUrl.value.trim(),
token: token.value.trim(),
conflictMode: conflictMode.value,
direction: direction.value,
...(resume && props.resumableJob ? { resumeJobId: props.resumableJob.id } : {})
})
token.value = ''
}
function startNew() {
submit(false)
}
function resumeSync() {
submit(true)
}
function setConflictMode(mode: 'skip' | 'overwrite') {
conflictMode.value = mode
}
function setDirection(value: SiteSyncDirection) {
direction.value = value
}
</script>
<template>
<article class="rounded-xl border border-default bg-elevated p-4 sm:p-5">
<div class="flex items-start gap-3">
<div class="flex size-10 shrink-0 items-center justify-center rounded-lg bg-primary/10 text-primary">
<UIcon
name="i-lucide-server-cog"
class="size-5"
/>
</div>
<div class="min-w-0">
<h3 class="text-base font-semibold">
{{ t('storage.backup.siteSyncTitle') }}
</h3>
<p class="mt-0.5 text-sm leading-relaxed text-muted">
{{ directionSubtitle }}
</p>
</div>
</div>
<div class="mt-4 grid gap-3 lg:grid-cols-[minmax(0,1.2fr)_minmax(0,1fr)_auto] lg:items-end">
<div class="space-y-1.5">
<label class="text-xs text-muted">{{ endpointLabel }}</label>
<UInput
v-model="targetBaseUrl"
placeholder="https://target.example.com"
size="sm"
class="w-full"
:disabled="busy || disabled"
/>
</div>
<div class="space-y-1.5">
<label class="text-xs text-muted">{{ tokenLabel }}</label>
<UInput
v-model="token"
type="password"
:placeholder="tokenPlaceholder"
size="sm"
class="w-full"
:disabled="busy || disabled"
/>
</div>
<UButton
:label="startLabel"
:icon="isPull ? 'i-lucide-cloud-download' : 'i-lucide-cloud-upload'"
color="primary"
:loading="busy"
:disabled="!canSubmit"
@click="startNew"
/>
<UButton
v-if="resumableJob"
:label="t('storage.backup.siteSyncResume')"
icon="i-lucide-play"
color="warning"
:loading="busy"
:disabled="!canResume"
class="lg:col-start-3"
@click="resumeSync"
/>
</div>
<div class="mt-3 grid gap-3 sm:grid-cols-2">
<div class="flex flex-wrap items-center gap-2">
<span class="text-xs text-muted">{{ t('storage.backup.siteSyncDirection') }}</span>
<UButton
size="xs"
:variant="direction === 'push' ? 'solid' : 'ghost'"
:color="direction === 'push' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncPush')"
:disabled="busy || disabled"
@click="setDirection('push')"
/>
<UButton
size="xs"
:variant="direction === 'pull' ? 'solid' : 'ghost'"
:color="direction === 'pull' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncPull')"
:disabled="busy || disabled"
@click="setDirection('pull')"
/>
</div>
<div class="flex flex-wrap items-center gap-2 sm:justify-end">
<span class="text-xs text-muted">{{ t('storage.backup.siteSyncConflict') }}</span>
<UButton
size="xs"
:variant="conflictMode === 'skip' ? 'solid' : 'ghost'"
:color="conflictMode === 'skip' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncSkip')"
:disabled="busy || disabled"
@click="setConflictMode('skip')"
/>
<UButton
size="xs"
:variant="conflictMode === 'overwrite' ? 'solid' : 'ghost'"
:color="conflictMode === 'overwrite' ? 'primary' : 'neutral'"
:label="t('storage.backup.siteSyncOverwrite')"
:disabled="busy || disabled"
@click="setConflictMode('overwrite')"
/>
</div>
</div>
<UAlert
v-if="activeJob"
class="mt-4"
color="primary"
variant="subtle"
icon="i-lucide-refresh-cw"
:description="activeJob.message"
/>
<div
v-if="activeJob"
class="mt-3 grid gap-3 rounded-lg border border-default bg-default/40 p-3 sm:grid-cols-2"
>
<div>
<p class="text-xs text-muted">
{{ t('storage.backup.siteSyncScanProgress') }}
</p>
<p class="mt-1 font-mono text-sm font-medium">
{{ activeJob.message }}
</p>
</div>
<div>
<p class="text-xs text-muted">
{{ t('storage.backup.siteSyncFileProgress') }}
</p>
<p class="mt-1 font-mono text-sm font-medium">
{{ fileProgress.synced }} / {{ fileProgress.total }}
</p>
</div>
</div>
<BackupJobProgress
v-if="activeJob"
class="mt-2"
:message="activeJob.message"
:progress="activeJob.progress"
:total="activeJob.total"
/>
</article>
</template>
+26 -1
View File
@@ -1,5 +1,9 @@
<script setup lang="ts"> <script setup lang="ts">
import type { BackupJobItem, SyncDraft } from '~/types/backup' import type {
BackupJobItem,
SiteSyncRequest,
SyncDraft
} from '~/types/backup'
import type { StorageBackendItem } from '~/types/storage' import type { StorageBackendItem } from '~/types/storage'
const props = defineProps<{ const props = defineProps<{
@@ -31,6 +35,7 @@ const {
confirmRestore, confirmRestore,
previewSync, previewSync,
startSync, startSync,
startSiteSync,
retryJob, retryJob,
cancelJob, cancelJob,
downloadUrl downloadUrl
@@ -45,6 +50,14 @@ const activeJob = computed(() => getActiveJob())
const exportBusy = computed(() => activeJob.value?.type === 'full_export') const exportBusy = computed(() => activeJob.value?.type === 'full_export')
const restoreBusy = computed(() => activeJob.value?.type === 'full_restore') const restoreBusy = computed(() => activeJob.value?.type === 'full_restore')
const syncBusy = computed(() => activeJob.value?.type === 'backend_sync') const syncBusy = computed(() => activeJob.value?.type === 'backend_sync')
const siteSyncBusy = computed(() => activeJob.value?.type === 'site_sync')
const resumableSiteSyncJob = computed(() =>
jobs.value.find(job =>
job.type === 'site_sync'
&& job.status === 'failed'
&& job.payload.resumable === true
) ?? null
)
const anyJobBusy = computed(() => Boolean(activeJob.value)) const anyJobBusy = computed(() => Boolean(activeJob.value))
onMounted(async () => { onMounted(async () => {
@@ -94,6 +107,10 @@ async function onSync(body: {
await startSync(body) await startSync(body)
} }
async function onSiteSync(body: SiteSyncRequest) {
await startSiteSync(body)
}
function onDownload(jobId: string) { function onDownload(jobId: string) {
window.open(downloadUrl(jobId), '_blank') window.open(downloadUrl(jobId), '_blank')
} }
@@ -159,6 +176,14 @@ function onViewJob(job: BackupJobItem) {
/> />
</div> </div>
<StorageSiteSyncCard
:busy="siteSyncBusy"
:disabled="anyJobBusy && !siteSyncBusy"
:active-job="siteSyncBusy ? activeJob : null"
:resumable-job="resumableSiteSyncJob"
@sync="onSiteSync"
/>
<StorageBackupJobTable <StorageBackupJobTable
:jobs="jobs" :jobs="jobs"
:polling="polling" :polling="polling"
+23 -1
View File
@@ -3,7 +3,8 @@ import type {
BackupJobItem, BackupJobItem,
BackupPackageItem, BackupPackageItem,
BackupRestorePreview, BackupRestorePreview,
BackendSyncPreview BackendSyncPreview,
SiteSyncRequest
} from '~/types/backup' } from '~/types/backup'
const jobs = ref<BackupJobItem[]>([]) const jobs = ref<BackupJobItem[]>([])
@@ -255,6 +256,26 @@ export function useBackup() {
} }
} }
async function startSiteSync(body: SiteSyncRequest) {
try {
const data = await $fetch<{ job: BackupJobItem }>(
'/api/admin/site-sync',
{
method: 'POST',
credentials: 'include',
body
}
)
await refreshJobsFirstPage()
startPolling()
toast.add({ title: t('storage.backup.siteSyncStarted'), color: 'success' })
return data.job
} catch (error: unknown) {
handleError(error, t('storage.backup.siteSyncFailed'))
throw error
}
}
async function retryJob(jobId: string) { async function retryJob(jobId: string) {
try { try {
await $fetch(`/api/admin/backup/jobs/${jobId}/retry`, { await $fetch(`/api/admin/backup/jobs/${jobId}/retry`, {
@@ -312,6 +333,7 @@ export function useBackup() {
confirmRestore, confirmRestore,
previewSync, previewSync,
startSync, startSync,
startSiteSync,
retryJob, retryJob,
cancelJob, cancelJob,
downloadUrl downloadUrl
+10 -1
View File
@@ -1,5 +1,6 @@
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed' export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
export type SiteSyncDirection = 'push' | 'pull'
export interface BackupJobItem { export interface BackupJobItem {
id: string id: string
@@ -51,3 +52,11 @@ export interface SyncDraft {
fromBackendId: string fromBackendId: string
toBackendId: string toBackendId: string
} }
export interface SiteSyncRequest {
targetBaseUrl: string
token: string
conflictMode: 'skip' | 'overwrite'
direction: SiteSyncDirection
resumeJobId?: string
}
+25
View File
@@ -645,6 +645,30 @@
"conflictSkip": "Skip existing files", "conflictSkip": "Skip existing files",
"conflictOverwrite": "Overwrite existing files and database", "conflictOverwrite": "Overwrite existing files and database",
"syncTitle": "Cross-backend sync", "syncTitle": "Cross-backend sync",
"siteSyncTitle": "Site-to-site sync",
"siteSyncSubtitle": "Sync media indexes, images, videos, files, thumbnails, HLS segments and tags to another PicHost. Matching usernames must already exist on the target.",
"siteSyncPushSubtitle": "Push media indexes, images, videos, files, thumbnails, HLS segments and tags to another PicHost. Matching usernames must already exist on the target.",
"siteSyncPullSubtitle": "Pull media indexes, images, videos, files, thumbnails, HLS segments and tags from another PicHost. Matching usernames must already exist on this site.",
"siteSyncTarget": "Target site URL",
"siteSyncSource": "Source site URL",
"siteSyncToken": "Target admin API token",
"siteSyncSourceToken": "Source admin API token",
"siteSyncTokenPlaceholder": "Enter target admin token",
"siteSyncSourceTokenPlaceholder": "Enter source admin token",
"siteSyncStart": "Start sync",
"siteSyncStartPush": "Start push",
"siteSyncStartPull": "Start pull",
"siteSyncDirection": "Sync direction:",
"siteSyncPush": "Push to target",
"siteSyncPull": "Pull from target",
"siteSyncConflict": "Conflict policy:",
"siteSyncSkip": "Skip existing",
"siteSyncOverwrite": "Overwrite existing",
"siteSyncResume": "Resume last sync",
"siteSyncScanProgress": "Media scan progress",
"siteSyncFileProgress": "Synced file count",
"siteSyncStarted": "Site-to-site sync started",
"siteSyncFailed": "Failed to start site-to-site sync",
"syncMigrateTitle": "Cross-backend migration", "syncMigrateTitle": "Cross-backend migration",
"syncMigrateSubtitle": "Migrate files between configured backends", "syncMigrateSubtitle": "Migrate files between configured backends",
"syncSubtitle": "Copy files between configured backends and update the index", "syncSubtitle": "Copy files between configured backends and update the index",
@@ -694,6 +718,7 @@
"jobTypeExport": "Site export", "jobTypeExport": "Site export",
"jobTypeRestore": "Site restore", "jobTypeRestore": "Site restore",
"jobTypeSync": "Cross-backend sync", "jobTypeSync": "Cross-backend sync",
"jobTypeSiteSync": "Site-to-site sync",
"statusPending": "Pending", "statusPending": "Pending",
"statusRunning": "Running", "statusRunning": "Running",
"statusDone": "Done", "statusDone": "Done",
+25
View File
@@ -645,6 +645,30 @@
"conflictSkip": "跳过已存在文件", "conflictSkip": "跳过已存在文件",
"conflictOverwrite": "覆盖已存在文件与数据库", "conflictOverwrite": "覆盖已存在文件与数据库",
"syncTitle": "跨后端同步", "syncTitle": "跨后端同步",
"siteSyncTitle": "两站点同步",
"siteSyncSubtitle": "将当前站点的媒体索引、图片、视频、文件、缩略图、HLS 分片和标签同步到另一台 PicHost。目标站点需已有同名用户。",
"siteSyncPushSubtitle": "将当前站点的媒体索引、图片、视频、文件、缩略图、HLS 分片和标签推送到另一台 PicHost。目标站点需已有同名用户。",
"siteSyncPullSubtitle": "从另一台 PicHost 拉取媒体索引、图片、视频、文件、缩略图、HLS 分片和标签。当前站点需已有同名用户。",
"siteSyncTarget": "目标站点地址",
"siteSyncSource": "来源站点地址",
"siteSyncToken": "目标站点管理员 API Token",
"siteSyncSourceToken": "来源站点管理员 API Token",
"siteSyncTokenPlaceholder": "输入目标站点管理员 Token",
"siteSyncSourceTokenPlaceholder": "输入来源站点管理员 Token",
"siteSyncStart": "开始同步",
"siteSyncStartPush": "开始推送",
"siteSyncStartPull": "开始拉取",
"siteSyncDirection": "同步策略:",
"siteSyncPush": "推送到目标站点",
"siteSyncPull": "从目标站点拉取",
"siteSyncConflict": "冲突策略:",
"siteSyncSkip": "跳过已存在",
"siteSyncOverwrite": "覆盖已存在",
"siteSyncResume": "继续上次同步",
"siteSyncScanProgress": "媒体扫描进度",
"siteSyncFileProgress": "已同步文件数量",
"siteSyncStarted": "两站点同步任务已开始",
"siteSyncFailed": "启动两站点同步失败",
"syncMigrateTitle": "跨后端迁移", "syncMigrateTitle": "跨后端迁移",
"syncMigrateSubtitle": "在已配置后端之间迁移文件", "syncMigrateSubtitle": "在已配置后端之间迁移文件",
"syncSubtitle": "在已配置的后端之间拷贝文件并更新索引", "syncSubtitle": "在已配置的后端之间拷贝文件并更新索引",
@@ -694,6 +718,7 @@
"jobTypeExport": "站点导出", "jobTypeExport": "站点导出",
"jobTypeRestore": "站点恢复", "jobTypeRestore": "站点恢复",
"jobTypeSync": "跨后端同步", "jobTypeSync": "跨后端同步",
"jobTypeSiteSync": "两站点同步",
"statusPending": "等待中", "statusPending": "等待中",
"statusRunning": "进行中", "statusRunning": "进行中",
"statusDone": "已完成", "statusDone": "已完成",
+59
View File
@@ -0,0 +1,59 @@
import { requireAdminAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { serializeBackupJob } from '../../utils/backup/api'
import {
resumeSiteSyncJob,
startSiteSyncJob
} from '../../utils/backup/runner'
interface SiteSyncBody {
targetBaseUrl?: string
token?: string
conflictMode?: 'skip' | 'overwrite'
direction?: 'push' | 'pull'
resumeJobId?: string
}
export default defineEventHandler(async (event) => {
const user = await requireAdminAuth(event)
const body = await readBody<SiteSyncBody>(event).catch(() => null)
const targetBaseUrl = body?.targetBaseUrl?.trim() ?? ''
const token = body?.token?.trim() ?? ''
const conflictMode = body?.conflictMode === 'skip' ? 'skip' : 'overwrite'
const direction = body?.direction === 'pull' ? 'pull' : 'push'
if (!token) {
createApiError(event, 'INVALID_REQUEST', '请输入目标站点管理员 API Token', 400)
}
try {
let job
if (body?.resumeJobId) {
job = resumeSiteSyncJob(body.resumeJobId, token, user.id)
} else {
let targetUrl: URL
try {
targetUrl = new URL(targetBaseUrl)
if (targetUrl.protocol !== 'http:' && targetUrl.protocol !== 'https:') {
throw new Error('invalid protocol')
}
} catch {
createApiError(event, 'INVALID_REQUEST', '请输入有效的目标站点地址', 400)
}
job = startSiteSyncJob({
targetBaseUrl: targetUrl.toString(),
conflictMode,
direction
}, token, user.id)
}
return { job: serializeBackupJob(job) }
} catch (error) {
if (error instanceof Error && error.message === 'BACKUP_JOB_RUNNING') {
createApiError(event, 'CONFLICT', '已有备份或迁移任务正在运行', 409)
}
if (error instanceof Error && error.message === 'SITE_SYNC_JOB_NOT_RESUMABLE') {
createApiError(event, 'INVALID_REQUEST', '该同步任务无法继续', 400)
}
throw error
}
})
+2 -2
View File
@@ -3,7 +3,7 @@ import { assertImageOwnership, requireApiOrAdminAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error' import { createApiError } from '../../utils/api-error'
import { logActivity } from '../../utils/activity-log' import { logActivity } from '../../utils/activity-log'
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index' import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
import { validateFileKey, validateVideoKey } from '../../utils/image-key' import { validateStorageKey, validateVideoKey } from '../../utils/image-key'
import { deleteImage, headImage } from '../../utils/storage' import { deleteImage, headImage } from '../../utils/storage'
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode' import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
import { MAX_DELETE_BATCH } from '../../utils/constants' import { MAX_DELETE_BATCH } from '../../utils/constants'
@@ -31,7 +31,7 @@ export default defineEventHandler(async (event) => {
const failed: BatchDeleteResponse['failed'] = [] const failed: BatchDeleteResponse['failed'] = []
for (const key of keys) { for (const key of keys) {
if (!validateFileKey(key) && !validateVideoKey(key)) { if (!validateStorageKey(key)) {
failed.push({ key, error: { code: 'INVALID_IMAGE_KEY', message: '无效的文件路径' } }) failed.push({ key, error: { code: 'INVALID_IMAGE_KEY', message: '无效的文件路径' } })
continue continue
} }
+2 -2
View File
@@ -3,7 +3,7 @@ import { assertImageOwnership, requireApiOrAdminAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error' import { createApiError } from '../../utils/api-error'
import { logActivity } from '../../utils/activity-log' import { logActivity } from '../../utils/activity-log'
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index' import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
import { validateFileKey, validateVideoKey } from '../../utils/image-key' import { validateStorageKey, validateVideoKey } from '../../utils/image-key'
import { deleteImage, headImage } from '../../utils/storage' import { deleteImage, headImage } from '../../utils/storage'
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode' import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
import { logInfo } from '../../utils/logger' import { logInfo } from '../../utils/logger'
@@ -20,7 +20,7 @@ export default defineEventHandler(async (event) => {
const body = await readBody<DeleteBody>(event).catch(() => null) const body = await readBody<DeleteBody>(event).catch(() => null)
const key = (typeof query.key === 'string' ? query.key : undefined) ?? body?.key const key = (typeof query.key === 'string' ? query.key : undefined) ?? body?.key
if (!key || (!validateFileKey(key) && !validateVideoKey(key))) { if (!key || !validateStorageKey(key)) {
createApiError(event, 'INVALID_IMAGE_KEY', '无效的文件路径', 400) createApiError(event, 'INVALID_IMAGE_KEY', '无效的文件路径', 400)
} }
+38
View File
@@ -0,0 +1,38 @@
import { requireSiteSyncAuth } from '../../utils/access'
import { listAllImageIndexRows } from '../../utils/image-index'
import { listUserIdUsernameMap } from '../../utils/db'
import { collectMediaAssetKeys } from '../../utils/media-assets'
import { getBackendForKey } from '../../utils/storage/resolver'
import type { SiteSyncInventory } from '../../utils/site-sync/protocol'
export default defineEventHandler(async (event): Promise<SiteSyncInventory> => {
await requireSiteSyncAuth(event)
const rows = listAllImageIndexRows()
const objects = new Map<string, number>()
for (const row of rows) {
const backend = await getBackendForKey(row.key)
let keys: string[]
try {
keys = await collectMediaAssetKeys(row, backend)
} catch {
keys = [
...(Number(row.source_deleted ?? 0) !== 1 ? [row.key] : []),
...(row.thumb_key ? [row.thumb_key] : []),
...(row.hls_key ? [row.hls_key] : [])
]
}
for (const key of new Set(keys)) {
const stored = await backend.head(key)
if (stored) objects.set(key, stored.size)
}
}
return {
users: [...listUserIdUsernameMap().values()].sort(),
mediaKeys: rows.map(row => row.key),
objects: [...objects].map(([key, size]) => ({ key, size }))
}
})
+75
View File
@@ -0,0 +1,75 @@
import { getQuery } from 'h3'
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { listUserIdUsernameMap } from '../../utils/db'
import {
getImageIndexRow,
getImageTotals,
listImageIndexRowsAfter
} from '../../utils/image-index'
import { validateStorageKey } from '../../utils/image-key'
import { collectMediaAssetKeys } from '../../utils/media-assets'
import { getBackendForKey } from '../../utils/storage/resolver'
import { getTagsForImageKeys } from '../../utils/tags'
import { toSiteSyncMediaTransfer } from '../../utils/site-sync/manifest'
import type {
SiteSyncMediaAsset,
SiteSyncMediaBundle,
SiteSyncMediaPage
} from '../../utils/site-sync/protocol'
export default defineEventHandler(async (event): Promise<SiteSyncMediaPage> => {
await requireSiteSyncAuth(event)
const query = getQuery(event)
const key = typeof query.key === 'string' ? query.key.trim() : ''
const after = typeof query.after === 'string' ? query.after : ''
const limit = Math.min(
100,
Math.max(1, Number.parseInt(String(query.limit ?? '50'), 10) || 50)
)
if (key && !validateStorageKey(key)) {
createApiError(event, 'INVALID_REQUEST', '媒体 key 无效', 400)
}
const rows = key
? [getImageIndexRow(key)].filter(row => row !== null)
: listImageIndexRowsAfter(after, limit)
if (key && rows.length === 0) {
createApiError(event, 'IMAGE_NOT_FOUND', '目标媒体不存在', 404)
}
const ownerMap = listUserIdUsernameMap()
const tagMap = getTagsForImageKeys(rows.map(row => row.key))
const items: SiteSyncMediaBundle[] = []
for (const row of rows) {
const backend = await getBackendForKey(row.key)
const keys = await collectMediaAssetKeys(row, backend)
const assets: SiteSyncMediaAsset[] = []
for (const assetKey of new Set(keys)) {
const stored = await backend.head(assetKey)
if (!stored) {
throw new Error(`目标媒体对象不存在:${assetKey}`)
}
assets.push({ key: assetKey, size: stored.size })
}
items.push({
media: toSiteSyncMediaTransfer(
row,
row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
tagMap.get(row.key) ?? []
),
assets
})
}
return {
items,
totalMedia: getImageTotals().count,
nextCursor: key || rows.length < limit
? null
: rows.at(-1)?.key ?? null
}
})
+26
View File
@@ -0,0 +1,26 @@
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { validateStorageKey } from '../../utils/image-key'
import { applySiteSyncMedia } from '../../utils/site-sync/apply'
import type { SiteSyncMediaTransfer } from '../../utils/site-sync/protocol'
interface Body {
media?: SiteSyncMediaTransfer
conflictMode?: 'skip' | 'overwrite'
}
export default defineEventHandler(async (event) => {
const admin = await requireSiteSyncAuth(event)
const body = await readBody<Body>(event).catch(() => null)
const media = body?.media
if (!media || !validateStorageKey(media.key)) {
createApiError(event, 'INVALID_REQUEST', '媒体同步参数无效', 400)
}
const conflictMode = body?.conflictMode === 'skip' ? 'skip' : 'overwrite'
try {
return await applySiteSyncMedia(media, conflictMode, admin.id)
} catch (error) {
const message = error instanceof Error ? error.message : '媒体同步失败'
createApiError(event, 'INVALID_REQUEST', message, 400)
}
})
+26
View File
@@ -0,0 +1,26 @@
import { getQuery, sendStream, setResponseHeader } from 'h3'
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { validateStorageKey } from '../../utils/image-key'
import { getBackendForKey } from '../../utils/storage/resolver'
export default defineEventHandler(async (event) => {
await requireSiteSyncAuth(event)
const query = getQuery(event)
const key = typeof query.key === 'string' ? query.key.trim() : ''
if (!validateStorageKey(key)) {
createApiError(event, 'INVALID_REQUEST', '同步对象 key 无效', 400)
}
const backend = await getBackendForKey(key)
const stored = await backend.head(key)
if (!stored) {
createApiError(event, 'IMAGE_NOT_FOUND', `目标对象不存在:${key}`, 404)
}
setResponseHeader(event, 'Content-Type', stored.contentType)
setResponseHeader(event, 'Content-Length', stored.size)
setResponseHeader(event, 'Cache-Control', 'no-store')
return sendStream(event, await backend.createStream(key))
})
+54
View File
@@ -0,0 +1,54 @@
import { readMultipartFormData } from 'h3'
import { requireSiteSyncAuth } from '../../utils/access'
import { createApiError } from '../../utils/api-error'
import { MAX_VIDEO_FILE_SIZE } from '../../utils/constants'
import { findUserByUsername } from '../../utils/db'
import { validateStorageKey } from '../../utils/image-key'
import { getActiveBackend } from '../../utils/storage/resolver'
function field(form: Awaited<ReturnType<typeof readMultipartFormData>>, name: string): string {
const part = form?.find(item => item.name === name)
return part?.data?.length ? new TextDecoder().decode(part.data).trim() : ''
}
export default defineEventHandler(async (event) => {
await requireSiteSyncAuth(event)
const form = await readMultipartFormData(event)
if (!form?.length) createApiError(event, 'INVALID_REQUEST', '未收到同步数据', 400)
const key = field(form, 'key')
const file = form.find(part => part.name === 'file' && part.data?.length)
if (!validateStorageKey(key) || !file?.data) {
createApiError(event, 'INVALID_REQUEST', '同步文件参数无效', 400)
}
if (file.data.length > MAX_VIDEO_FILE_SIZE) {
createApiError(event, 'FILE_TOO_LARGE', '同步文件超过 500 MB', 413)
}
const conflictMode = field(form, 'conflictMode') === 'skip' ? 'skip' : 'overwrite'
const backend = await getActiveBackend()
if (conflictMode === 'skip' && await backend.head(key)) {
return { success: true, skipped: true }
}
const ownerUsername = field(form, 'ownerUsername')
const owner = ownerUsername ? findUserByUsername(ownerUsername) : null
if (ownerUsername && !owner) {
createApiError(event, 'INVALID_REQUEST', `目标站点缺少用户:${ownerUsername}`, 400)
}
await backend.put(key, new Uint8Array(file.data), {
originalName: field(form, 'originalName') || file.filename || key.split('/').pop() || 'file',
uploadedAt: field(form, 'uploadedAt') || new Date().toISOString(),
contentType: file.type || field(form, 'contentType') || 'application/octet-stream',
size: file.data.length,
userId: owner?.id ?? null,
mediaType: key.startsWith('videos/')
? 'video'
: key.startsWith('files/')
? 'file'
: 'image'
})
return { success: true, skipped: false }
})
+10
View File
@@ -0,0 +1,10 @@
import { requireSiteSyncAuth } from '../../utils/access'
import { listUserIdUsernameMap } from '../../utils/db'
import type { SiteSyncUsers } from '../../utils/site-sync/protocol'
export default defineEventHandler(async (event): Promise<SiteSyncUsers> => {
await requireSiteSyncAuth(event)
return {
users: [...listUserIdUsernameMap().values()].sort()
}
})
+31 -1
View File
@@ -1,5 +1,5 @@
import type { H3Event } from 'h3' import type { H3Event } from 'h3'
import { findUserByApiToken, getPrimaryAdminUserId } from './db' import { findUserById, findUserByApiToken, getPrimaryAdminUserId } from './db'
import { getAdminSecret, getApiUploadToken } from './env' import { getAdminSecret, getApiUploadToken } from './env'
import { createApiError } from './api-error' import { createApiError } from './api-error'
import { import {
@@ -110,6 +110,36 @@ export function resolveUserIdFromApiToken(
return null return null
} }
/** 两站点同步:全站 Token 或管理员个人 Token;也允许管理员网页会话。 */
export async function requireSiteSyncAuth(event: H3Event): Promise<AuthUser> {
if (devBypass(event)) {
return { id: 0, username: 'dev', role: 'admin' }
}
if (verifyGlobalApiUploadToken(event)) {
const adminId = getPrimaryAdminUserId()
if (adminId != null) {
const admin = findUserById(adminId)
if (admin) {
return { id: admin.id, username: admin.username, role: admin.role }
}
}
}
const userId = resolveUserIdFromApiToken(event)
if (userId != null) {
const user = findUserById(userId)
if (user?.role === 'admin') {
return { id: user.id, username: user.username, role: user.role }
}
}
const sessionUser = await getCurrentUser(event)
if (sessionUser?.role === 'admin') return sessionUser
createApiError(event, 'FORBIDDEN', '需要目标站点管理员同步 Token', 403)
}
export function verifyApiUploadToken(event: H3Event): boolean { export function verifyApiUploadToken(event: H3Event): boolean {
return verifyGlobalApiUploadToken(event) return verifyGlobalApiUploadToken(event)
|| resolveUserIdFromApiToken(event) !== null || resolveUserIdFromApiToken(event) !== null
+2
View File
@@ -9,6 +9,8 @@ export {
startFullExportJob, startFullExportJob,
startFullRestoreJob, startFullRestoreJob,
startBackendSyncJob, startBackendSyncJob,
startSiteSyncJob,
resumeSiteSyncJob,
isBackupRunnerBusy, isBackupRunnerBusy,
cancelBackupJob cancelBackupJob
} from './runner' } from './runner'
+1 -1
View File
@@ -2,7 +2,7 @@ import { randomBytes } from 'node:crypto'
import { getDb, getPrimaryAdminUserId } from '../db' import { getDb, getPrimaryAdminUserId } from '../db'
import { ensureBackupSchema } from './schema' import { ensureBackupSchema } from './schema'
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed' export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
export interface BackupJob { export interface BackupJob {
+53
View File
@@ -1,6 +1,7 @@
import { runBackendSyncJob } from './backend-sync' import { runBackendSyncJob } from './backend-sync'
import { runFullExportJob } from './full-export' import { runFullExportJob } from './full-export'
import { runFullRestoreJob } from './full-restore' import { runFullRestoreJob } from './full-restore'
import { registerSiteSyncToken, runSiteSyncJob, type SiteSyncOptions } from '../site-sync/runner'
import { import {
createBackupJob, createBackupJob,
getBackupJob, getBackupJob,
@@ -114,3 +115,55 @@ export function startBackendSyncJob(options: {
scheduleBackupJob(job, () => runBackendSyncJob(job.id, options)) scheduleBackupJob(job, () => runBackendSyncJob(job.id, options))
return getBackupJob(job.id)! return getBackupJob(job.id)!
} }
export function startSiteSyncJob(
options: SiteSyncOptions,
token: string,
userId?: number | null
): BackupJob {
claimJobSlot()
const job = createBackupJob({
type: 'site_sync',
userId,
message: '等待开始…',
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: options.direction
}
})
registerSiteSyncToken(job.id, token)
scheduleBackupJob(job, () => runSiteSyncJob(job.id, options))
return getBackupJob(job.id)!
}
export function resumeSiteSyncJob(
jobId: string,
token: string,
userId?: number | null
): BackupJob {
claimJobSlot()
const job = getBackupJob(jobId, userId ?? undefined)
if (
!job
|| job.type !== 'site_sync'
|| job.status !== 'failed'
|| job.payload.resumable !== true
) {
throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
}
const targetBaseUrl = typeof job.payload.targetBaseUrl === 'string'
? job.payload.targetBaseUrl
: ''
const conflictMode = job.payload.conflictMode === 'skip' ? 'skip' : 'overwrite'
const direction = job.payload.direction === 'pull' ? 'pull' : 'push'
if (!targetBaseUrl) throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
registerSiteSyncToken(job.id, token)
scheduleBackupJob(job, () => runSiteSyncJob(job.id, {
targetBaseUrl,
conflictMode,
direction
}))
return getBackupJob(job.id)!
}
+41 -1
View File
@@ -1,11 +1,50 @@
import { getDb } from '../db' import { getDb } from '../db'
function backupJobsTableSql(): string {
const row = getDb().prepare(`
SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'backup_jobs'
`).get() as { sql?: string } | undefined
return row?.sql ?? ''
}
function migrateBackupJobTypes(): void {
if (backupJobsTableSql().includes('\'site_sync\'')) return
const db = getDb()
db.exec(`
CREATE TABLE backup_jobs_new (
id TEXT PRIMARY KEY,
user_id INTEGER,
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
progress INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0,
message TEXT NOT NULL DEFAULT '',
payload_json TEXT NOT NULL DEFAULT '{}',
output_path TEXT,
error TEXT,
created_at TEXT NOT NULL,
finished_at TEXT
);
INSERT INTO backup_jobs_new
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
SELECT id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at
FROM backup_jobs;
DROP TABLE backup_jobs;
ALTER TABLE backup_jobs_new RENAME TO backup_jobs;
CREATE INDEX IF NOT EXISTS idx_backup_jobs_created_at
ON backup_jobs(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_backup_jobs_user_created
ON backup_jobs(user_id, created_at DESC);
`)
}
export function ensureBackupSchema(): void { export function ensureBackupSchema(): void {
getDb().exec(` getDb().exec(`
CREATE TABLE IF NOT EXISTS backup_jobs ( CREATE TABLE IF NOT EXISTS backup_jobs (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
user_id INTEGER, user_id INTEGER,
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync')), type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')), status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
progress INTEGER NOT NULL DEFAULT 0, progress INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL DEFAULT 0,
@@ -36,4 +75,5 @@ export function ensureBackupSchema(): void {
WHERE user_id IS NULL WHERE user_id IS NULL
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin'); AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
`) `)
migrateBackupJobTypes()
} }
+23 -7
View File
@@ -35,6 +35,7 @@ export interface ImageIndexRow {
backend_id: string backend_id: string
user_id: number | null user_id: number | null
folder: string folder: string
media_type: MediaType
original_name: string original_name: string
content_type: string content_type: string
size: number size: number
@@ -71,6 +72,7 @@ function rowToStoredImage(row: ImageIndexRow): StoredImage {
contentType: row.content_type, contentType: row.content_type,
size: row.size, size: row.size,
userId: row.user_id, userId: row.user_id,
mediaType: row.media_type,
hlsKey: row.hls_key ?? null, hlsKey: row.hls_key ?? null,
transcodeStatus: (row.transcode_status ?? 'none') as TranscodeStatus, transcodeStatus: (row.transcode_status ?? 'none') as TranscodeStatus,
transcodeProgress: Number(row.transcode_progress ?? 0), transcodeProgress: Number(row.transcode_progress ?? 0),
@@ -232,7 +234,7 @@ export async function purgeOrphanImageIndexRows(): Promise<number> {
export function getImageIndexRow(key: string): ImageIndexRow | null { export function getImageIndexRow(key: string): ImageIndexRow | null {
ensureStorageSchema() ensureStorageSchema()
const row = getDb().prepare(` const row = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images FROM images
WHERE key = ? WHERE key = ?
`).get(key) as ImageIndexRow | undefined `).get(key) as ImageIndexRow | undefined
@@ -425,16 +427,30 @@ export function getImageTotals(): { count: number, bytes: number } {
export function listAllImageIndexRows(): ImageIndexRow[] { export function listAllImageIndexRows(): ImageIndexRow[] {
ensureStorageSchema() ensureStorageSchema()
return getDb().prepare(` return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images FROM images
ORDER BY key ASC ORDER BY key ASC
`).all() as unknown as ImageIndexRow[] `).all() as unknown as ImageIndexRow[]
} }
export function listImageIndexRowsAfter(
afterKey: string,
limit: number
): ImageIndexRow[] {
ensureStorageSchema()
return getDb().prepare(`
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images
WHERE key > ?
ORDER BY key ASC
LIMIT ?
`).all(afterKey, limit) as unknown as ImageIndexRow[]
}
export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] { export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
ensureStorageSchema() ensureStorageSchema()
return getDb().prepare(` return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images FROM images
WHERE user_id = ? WHERE user_id = ?
ORDER BY key ASC ORDER BY key ASC
@@ -444,7 +460,7 @@ export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
export function listImageIndexRowsByBackend(backendId: string): ImageIndexRow[] { export function listImageIndexRowsByBackend(backendId: string): ImageIndexRow[] {
ensureStorageSchema() ensureStorageSchema()
return getDb().prepare(` return getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images FROM images
WHERE backend_id = ? WHERE backend_id = ?
ORDER BY key ASC ORDER BY key ASC
@@ -687,7 +703,7 @@ export async function listImages(options: {
const backendSql = buildBackendFilterSql(options.backendId, params) const backendSql = buildBackendFilterSql(options.backendId, params)
const tagSql = buildTagFilterSql(options.tagFilter, params) const tagSql = buildTagFilterSql(options.tagFilter, params)
const rows = getDb().prepare(` const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images FROM images
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql} WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
ORDER BY uploaded_at DESC, key DESC ORDER BY uploaded_at DESC, key DESC
@@ -733,7 +749,7 @@ export async function searchImages(options: {
const backendSql = buildBackendFilterSql(options.backendId, params) const backendSql = buildBackendFilterSql(options.backendId, params)
const tagSql = buildTagFilterSql(options.tagFilter, params) const tagSql = buildTagFilterSql(options.tagFilter, params)
const rows = getDb().prepare(` const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images FROM images
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql} WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
ORDER BY uploaded_at DESC, key DESC ORDER BY uploaded_at DESC, key DESC
@@ -867,7 +883,7 @@ export async function getUserScopedStorageStats(
export async function listImageKeysForAutoDelete(): Promise<StoredImage[]> { export async function listImageKeysForAutoDelete(): Promise<StoredImage[]> {
ensureStorageSchema() ensureStorageSchema()
const rows = getDb().prepare(` const rows = getDb().prepare(`
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
FROM images FROM images
WHERE media_type = 'image' WHERE media_type = 'image'
ORDER BY uploaded_at ASC ORDER BY uploaded_at ASC
+1
View File
@@ -8,6 +8,7 @@ const row: ImageIndexRow = {
backend_id: 'local', backend_id: 'local',
user_id: 1, user_id: 1,
folder: 'videos', folder: 'videos',
media_type: 'video',
original_name: 'example.mp4', original_name: 'example.mp4',
content_type: 'video/mp4', content_type: 'video/mp4',
size: 100, size: 100,
+79
View File
@@ -0,0 +1,79 @@
import { findUserByUsername } from '../db'
import {
getImageIndexRow,
insertImageIndex,
updateImageThumb,
updateImageTranscode
} from '../image-index'
import { getActiveBackend } from '../storage/resolver'
import {
createTag,
findTagByName,
setImageTags
} from '../tags'
import type { SiteSyncMediaTransfer } from './protocol'
export async function applySiteSyncMedia(
media: SiteSyncMediaTransfer,
conflictMode: 'skip' | 'overwrite',
actorUserId: number
): Promise<{ success: true, skipped: boolean }> {
if (conflictMode === 'skip' && getImageIndexRow(media.key)) {
return { success: true, skipped: true }
}
const owner = media.ownerUsername
? findUserByUsername(media.ownerUsername)
: null
if (media.ownerUsername && !owner) {
throw new Error(`当前站点缺少用户:${media.ownerUsername}`)
}
const backend = await getActiveBackend()
const requiredKeys = [
...(!media.sourceDeleted ? [media.key] : []),
...(media.thumbKey ? [media.thumbKey] : []),
...(media.hlsKey ? [media.hlsKey] : [])
]
for (const key of requiredKeys) {
if (!await backend.head(key)) {
throw new Error(`同步对象尚未写入:${key}`)
}
}
insertImageIndex({
key: media.key,
backendId: backend.id,
userId: owner?.id ?? null,
originalName: media.originalName,
contentType: media.contentType,
size: media.size,
uploadedAt: media.uploadedAt,
mediaType: media.mediaType
})
if (media.thumbKey) updateImageThumb(media.key, media.thumbKey)
if (media.mediaType === 'video') {
updateImageTranscode(media.key, {
status: media.transcodeStatus,
hlsKey: media.hlsKey,
progress: media.transcodeProgress,
preset: media.transcodePreset,
sourceDeleted: media.sourceDeleted,
error: media.transcodeError
})
}
const tagOwnerId = owner?.id ?? actorUserId
const tagIds: number[] = []
for (const tag of media.tags) {
const existing = findTagByName(tagOwnerId, tag.name)
const item = existing
? { id: existing.id }
: createTag(tagOwnerId, tag.name, tag.color)
tagIds.push(item.id)
}
setImageTags(media.key, tagIds, tagOwnerId, true)
return { success: true, skipped: false }
}
+38
View File
@@ -0,0 +1,38 @@
import type { ImageIndexRow } from '../image-index'
import { getTagsForImageKeys } from '../tags'
import type {
SiteSyncMediaTransfer,
SiteSyncTag
} from './protocol'
export function toSiteSyncMediaTransfer(
row: ImageIndexRow,
ownerUsername: string | null,
tags: SiteSyncTag[]
): SiteSyncMediaTransfer {
return {
key: row.key,
ownerUsername,
originalName: row.original_name,
contentType: row.content_type,
size: row.size,
uploadedAt: row.uploaded_at,
mediaType: row.media_type,
thumbKey: row.thumb_key,
hlsKey: row.hls_key,
transcodeStatus: row.transcode_status as SiteSyncMediaTransfer['transcodeStatus'],
transcodeProgress: Number(row.transcode_progress ?? 0),
transcodePreset: row.transcode_preset,
transcodeError: row.transcode_error,
sourceDeleted: Number(row.source_deleted ?? 0) === 1,
tags
}
}
export function toSiteSyncMediaTransferWithTags(
row: ImageIndexRow,
ownerUsername: string | null
): SiteSyncMediaTransfer {
const tags = getTagsForImageKeys([row.key]).get(row.key) ?? []
return toSiteSyncMediaTransfer(row, ownerUsername, tags)
}
+59
View File
@@ -0,0 +1,59 @@
import type { MediaType, TranscodeStatus } from '../storage/types'
export interface SiteSyncTag {
name: string
color: string
}
export interface SiteSyncMediaTransfer {
key: string
ownerUsername: string | null
originalName: string
contentType: string
size: number
uploadedAt: string
mediaType: MediaType
thumbKey: string | null
hlsKey: string | null
transcodeStatus: TranscodeStatus
transcodeProgress: number
transcodePreset: string | null
transcodeError: string | null
sourceDeleted: boolean
tags: SiteSyncTag[]
}
export interface SiteSyncInventory {
users: string[]
mediaKeys: string[]
objects: Array<{ key: string, size: number }>
}
export interface SiteSyncMediaAsset {
key: string
size: number
}
export interface SiteSyncMediaBundle {
media: SiteSyncMediaTransfer
assets: SiteSyncMediaAsset[]
}
export interface SiteSyncMediaPage {
items: SiteSyncMediaBundle[]
totalMedia: number
nextCursor: string | null
}
export interface SiteSyncUsers {
users: string[]
}
export type SiteSyncDirection = 'push' | 'pull'
export interface SiteSyncObjectOptions {
targetBaseUrl: string
token: string
conflictMode: 'skip' | 'overwrite'
direction?: SiteSyncDirection
}
+55
View File
@@ -0,0 +1,55 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { fetchSiteSyncJson, retrySiteSyncOperation } from './runner'
afterEach(() => {
vi.unstubAllGlobals()
})
describe('fetchSiteSyncJson', () => {
it('retries a broken connection up to three attempts', async () => {
const fetchMock = vi.fn()
.mockRejectedValueOnce(new Error('connection reset'))
.mockRejectedValueOnce(new Error('connection reset'))
.mockResolvedValueOnce(new Response('{"ok":true}', {
status: 200,
headers: { 'Content-Type': 'application/json' }
}))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).resolves.toEqual({ ok: true })
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('retries client errors because every sync failure gets three attempts', async () => {
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
new Response('{"message":"invalid"}', {
status: 400,
headers: { 'Content-Type': 'application/json' }
})
))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('invalid')
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('fails after three server errors', async () => {
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
new Response('{"message":"unavailable"}', {
status: 503,
headers: { 'Content-Type': 'application/json' }
})
))
vi.stubGlobal('fetch', fetchMock)
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('unavailable')
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('retries local sync failures and stops after the third attempt', async () => {
const operation = vi.fn().mockRejectedValue(new Error('database is busy'))
await expect(retrySiteSyncOperation(operation)).rejects.toThrow('database is busy')
expect(operation).toHaveBeenCalledTimes(3)
})
})
+641
View File
@@ -0,0 +1,641 @@
import { basename } from 'node:path'
import { contentTypeFromKey } from '../content-type'
import {
getImageIndexRow,
listAllImageIndexRows,
type ImageIndexRow
} from '../image-index'
import {
findUserByUsername,
getPrimaryAdminUserId,
listUserIdUsernameMap
} from '../db'
import { collectMediaAssetKeys } from '../media-assets'
import {
getActiveBackend,
getBackendForKey
} from '../storage/resolver'
import type { StorageBackend } from '../storage/types'
import {
getBackupJob,
markJobDone,
markJobFailed,
updateBackupJob
} from '../backup/jobs'
import { applySiteSyncMedia } from './apply'
import { toSiteSyncMediaTransferWithTags } from './manifest'
import type {
SiteSyncDirection,
SiteSyncInventory,
SiteSyncMediaBundle,
SiteSyncMediaPage,
SiteSyncUsers
} from './protocol'
export interface SiteSyncOptions {
targetBaseUrl: string
conflictMode: 'skip' | 'overwrite'
direction: SiteSyncDirection
}
const tokens = (globalThis as typeof globalThis & {
__pichostSiteSyncTokens?: Map<string, string>
}).__pichostSiteSyncTokens ?? new Map<string, string>()
;(globalThis as typeof globalThis & {
__pichostSiteSyncTokens?: Map<string, string>
}).__pichostSiteSyncTokens = tokens
export function registerSiteSyncToken(jobId: string, token: string): void {
tokens.set(jobId, token)
}
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = []
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
}
return Buffer.concat(chunks)
}
function targetUrl(baseUrl: string, path: string): string {
return `${baseUrl.replace(/\/+$/, '')}${path}`
}
async function responseError(response: Response): Promise<Error> {
const payload = await response.json().catch(() => null) as {
message?: string
statusMessage?: string
} | null
return new Error(payload?.message || payload?.statusMessage || `目标站点返回 ${response.status}`)
}
const SITE_SYNC_MAX_ATTEMPTS = 3
export async function retrySiteSyncOperation<T>(
operation: () => Promise<T>
): Promise<T> {
let lastError: Error | null = null
for (let attempt = 1; attempt <= SITE_SYNC_MAX_ATTEMPTS; attempt++) {
try {
return await operation()
} catch (error) {
lastError = error instanceof Error ? error : new Error('两站点同步失败')
if (attempt < SITE_SYNC_MAX_ATTEMPTS) {
await new Promise(resolve => setTimeout(resolve, attempt * 500))
}
}
}
throw lastError ?? new Error('两站点同步失败')
}
async function fetchSiteSyncJsonOnce<T>(
url: string,
token: string,
init?: RequestInit
): Promise<T> {
const response = await fetch(url, {
...init,
headers: {
'Auth-Token': token,
...(init?.headers ?? {})
}
})
if (!response.ok) throw await responseError(response)
return await response.json() as T
}
export async function fetchSiteSyncJson<T>(
url: string,
token: string,
init?: RequestInit
): Promise<T> {
return retrySiteSyncOperation(() => (
fetchSiteSyncJsonOnce<T>(url, token, init)
))
}
async function uploadObject(
options: SiteSyncOptions,
token: string,
backend: StorageBackend,
row: ImageIndexRow,
ownerUsername: string | null,
key: string
): Promise<void> {
const stored = await backend.head(key)
if (!stored) throw new Error(`源文件不存在:${key}`)
const bytes = await streamToBuffer(await backend.createStream(key))
const form = new FormData()
const isPrimary = key === row.key
form.append('key', key)
form.append('conflictMode', options.conflictMode)
form.append('ownerUsername', ownerUsername ?? '')
form.append('originalName', isPrimary ? row.original_name : basename(key))
form.append('uploadedAt', row.uploaded_at)
const contentType = isPrimary ? row.content_type : contentTypeFromKey(key)
form.append('contentType', contentType)
form.append(
'file',
new Blob([new Uint8Array(bytes)], { type: contentType }),
basename(key)
)
await fetchSiteSyncJsonOnce<{ success: boolean }>(
targetUrl(options.targetBaseUrl, '/api/site-sync/object'),
token,
{ method: 'POST', body: form }
)
}
async function downloadSiteSyncObject(
options: SiteSyncOptions,
token: string,
key: string
): Promise<Buffer> {
const response = await fetch(
targetUrl(
options.targetBaseUrl,
`/api/site-sync/object?key=${encodeURIComponent(key)}`
),
{
headers: {
'Auth-Token': token
}
}
)
if (!response.ok) throw await responseError(response)
return Buffer.from(await response.arrayBuffer())
}
async function runSiteSyncPull(
jobId: string,
options: SiteSyncOptions,
token: string
): Promise<void> {
const job = getBackupJob(jobId)
const checkpoint = job?.payload ?? {}
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
? checkpoint.lastMediaKey
: ''
let scanned = Number(checkpoint.scanned ?? 0)
let added = Number(checkpoint.added ?? 0)
let skipped = Number(checkpoint.skipped ?? 0)
let filesSynced = Number(checkpoint.filesSynced ?? 0)
let totalFiles = Number(checkpoint.totalFiles ?? 0)
let totalMedia = Number(checkpoint.totalMedia ?? job?.total ?? 0)
const failedKeys = new Set(
Array.isArray(checkpoint.failedKeys)
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
: []
)
const saveCheckpoint = (message: string) => {
updateBackupJob(jobId, {
progress: scanned,
total: totalMedia,
message,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'pull',
lastMediaKey,
scanned,
added,
skipped,
failed: failedKeys.size,
filesSynced,
totalFiles,
totalMedia,
failedKeys: [...failedKeys],
resumable: true
}
})
}
const progressMessage = (suffix = '') => (
`扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}${suffix}`
)
const failAndCheckpoint = (error: unknown) => {
const message = progressMessage(' · 已保存断点')
saveCheckpoint(message)
markJobFailed(
jobId,
error instanceof Error ? error.message : '从目标站点拉取失败',
message
)
}
try {
const targetUsers = new Set(
(await fetchSiteSyncJson<SiteSyncUsers>(
targetUrl(options.targetBaseUrl, '/api/site-sync/users'),
token
)).users
)
const localUserMap = listUserIdUsernameMap()
const localUsers = new Set(localUserMap.values())
const missingUsers = [...targetUsers].filter(name => !localUsers.has(name))
if (missingUsers.length) {
throw new Error(`当前站点缺少同名用户:${missingUsers.join('、')}`)
}
const backend = await retrySiteSyncOperation(() => getActiveBackend())
const actorUserId = job?.userId ?? getPrimaryAdminUserId() ?? 0
const readMediaPage = (path: string) => fetchSiteSyncJson<SiteSyncMediaPage>(
targetUrl(options.targetBaseUrl, path),
token
)
const processBundleOnce = async (
bundle: SiteSyncMediaBundle,
isRetry: boolean
): Promise<'added' | 'skipped'> => {
const { media, assets } = bundle
if (
!isRetry
&& options.conflictMode === 'skip'
&& getImageIndexRow(media.key)
) {
return 'skipped'
}
const owner = media.ownerUsername
? findUserByUsername(media.ownerUsername)
: null
for (const asset of assets) {
if (
options.conflictMode === 'skip'
&& await backend.head(asset.key)
) {
continue
}
const bytes = await downloadSiteSyncObject(
options,
token,
asset.key
)
if (bytes.length !== asset.size) {
throw new Error(
`同步对象大小不一致:${asset.key},预期 ${asset.size},实际 ${bytes.length}`
)
}
const isPrimary = asset.key === media.key
await backend.put(asset.key, new Uint8Array(bytes), {
originalName: isPrimary ? media.originalName : basename(asset.key),
uploadedAt: media.uploadedAt,
contentType: isPrimary
? media.contentType
: contentTypeFromKey(asset.key),
size: bytes.length,
userId: owner?.id ?? null,
mediaType: isPrimary
? media.mediaType
: asset.key.startsWith('videos/')
? 'video'
: asset.key.startsWith('files/')
? 'file'
: 'image'
})
}
await applySiteSyncMedia(media, 'overwrite', actorUserId)
return 'added'
}
const processBundle = (bundle: SiteSyncMediaBundle, isRetry: boolean) => (
retrySiteSyncOperation(() => processBundleOnce(bundle, isRetry))
)
const recordOutcome = (outcome: 'added' | 'skipped', fileCount: number) => {
if (outcome === 'added') added += 1
else skipped += 1
filesSynced += fileCount
totalFiles = Math.max(totalFiles, filesSynced)
}
for (const key of [...failedKeys]) {
try {
const page = await readMediaPage(
`/api/site-sync/media?key=${encodeURIComponent(key)}`
)
const bundle = page.items[0]
if (!bundle) throw new Error(`目标媒体不存在:${key}`)
const outcome = await processBundle(bundle, true)
recordOutcome(outcome, bundle.assets.length)
failedKeys.delete(key)
if (key > lastMediaKey) lastMediaKey = key
saveCheckpoint(progressMessage())
} catch (error) {
failedKeys.add(key)
failAndCheckpoint(error)
return
}
}
let cursor = lastMediaKey
let lastCheckpointAt = 0
while (true) {
let page: SiteSyncMediaPage
try {
page = await readMediaPage(
`/api/site-sync/media?limit=50${cursor ? `&after=${encodeURIComponent(cursor)}` : ''}`
)
} catch (error) {
failAndCheckpoint(error)
return
}
totalMedia = Math.max(totalMedia, page.totalMedia)
for (const bundle of page.items) {
scanned += 1
try {
const outcome = await processBundle(bundle, false)
recordOutcome(outcome, bundle.assets.length)
failedKeys.delete(bundle.media.key)
} catch (error) {
failedKeys.add(bundle.media.key)
lastMediaKey = bundle.media.key
failAndCheckpoint(error)
return
}
lastMediaKey = bundle.media.key
const now = Date.now()
if (
now - lastCheckpointAt >= 1000
|| scanned === totalMedia
|| !page.nextCursor
) {
saveCheckpoint(progressMessage())
lastCheckpointAt = now
}
}
if (!page.nextCursor) break
cursor = page.nextCursor
}
if (failedKeys.size) {
markJobFailed(
jobId,
`${failedKeys.size} 条媒体拉取失败`,
progressMessage()
)
saveCheckpoint(progressMessage())
return
}
totalFiles = filesSynced
markJobDone(jobId, {
message: `扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'pull',
lastMediaKey,
scanned,
added,
skipped,
failed: 0,
filesSynced,
totalFiles,
totalMedia: totalMedia || scanned,
failedKeys: [],
resumable: false
}
})
} catch (error) {
failAndCheckpoint(error)
}
}
export async function runSiteSyncJob(
jobId: string,
options: SiteSyncOptions
): Promise<void> {
const token = tokens.get(jobId)
if (!token) throw new Error('同步 Token 不可用,请重新发起任务')
try {
if (options.direction === 'pull') {
await runSiteSyncPull(jobId, options, token)
return
}
const inventory = await fetchSiteSyncJson<SiteSyncInventory>(
targetUrl(options.targetBaseUrl, '/api/site-sync/inventory'),
token
)
const targetUsers = new Set(inventory.users)
const targetMedia = new Set(inventory.mediaKeys)
const targetObjects = new Map(inventory.objects.map(item => [item.key, item.size]))
const rows = listAllImageIndexRows()
const ownerMap = listUserIdUsernameMap()
const job = getBackupJob(jobId)
const checkpoint = job?.payload ?? {}
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
? checkpoint.lastMediaKey
: ''
let scanned = Number(checkpoint.scanned ?? 0)
let added = Number(checkpoint.added ?? 0)
let skipped = Number(checkpoint.skipped ?? 0)
let filesSynced = Number(checkpoint.filesSynced ?? 0)
const failedKeys = new Set(
Array.isArray(checkpoint.failedKeys)
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
: []
)
const missingUsers = [...new Set(
rows
.map(row => row.user_id == null ? null : ownerMap.get(row.user_id))
.filter((name): name is string => Boolean(name))
)].filter(name => !targetUsers.has(name))
if (missingUsers.length) {
throw new Error(`目标站点缺少同名用户:${missingUsers.join('、')}`)
}
const plans: Array<{
row: ImageIndexRow
ownerUsername: string | null
backend: StorageBackend
keys: string[]
}> = []
let totalFiles = 0
updateBackupJob(jobId, {
total: rows.length,
progress: scanned,
message: '正在统计同步文件…'
})
for (const row of rows) {
const backend = await getBackendForKey(row.key)
let assetKeys: string[]
try {
assetKeys = await collectMediaAssetKeys(row, backend)
} catch {
assetKeys = [
...(!Number(row.source_deleted ?? 0) ? [row.key] : []),
...(row.thumb_key ? [row.thumb_key] : []),
...(row.hls_key ? [row.hls_key] : [])
]
}
const keys = [...new Set(assetKeys)]
totalFiles += keys.length
plans.push({
row,
ownerUsername: row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
backend,
keys
})
}
const saveCheckpoint = (message: string) => {
updateBackupJob(jobId, {
progress: scanned,
total: rows.length,
message,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'push',
lastMediaKey,
scanned,
added,
skipped,
failed: failedKeys.size,
filesSynced,
totalFiles,
failedKeys: [...failedKeys],
resumable: true
}
})
}
plans.sort((a, b) => (
a.row.key < b.row.key
? -1
: a.row.key > b.row.key
? 1
: 0
))
lastMediaKey = lastMediaKey || ''
let lastCheckpointAt = 0
const processPlanOnce = async (plan: typeof plans[number], isRetry: boolean) => {
const { row, ownerUsername, backend, keys } = plan
if (
!isRetry
&& options.conflictMode === 'skip'
&& targetMedia.has(row.key)
) {
skipped += 1
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
failedKeys.delete(row.key)
return
}
for (const key of keys) {
const stored = await backend.head(key)
if (!stored) throw new Error(`源文件不存在:${key}`)
if (
options.conflictMode === 'skip'
&& targetObjects.get(key) === stored.size
) {
continue
}
await uploadObject(options, token, backend, row, ownerUsername, key)
}
await fetchSiteSyncJsonOnce<{ success: boolean }>(
targetUrl(options.targetBaseUrl, '/api/site-sync/media'),
token,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
media: toSiteSyncMediaTransferWithTags(row, ownerUsername),
conflictMode: options.conflictMode
})
}
)
failedKeys.delete(row.key)
added += 1
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
}
const processPlan = (
plan: typeof plans[number],
isRetry: boolean
) => retrySiteSyncOperation(() => processPlanOnce(plan, isRetry))
const retryPlans = plans.filter(plan => failedKeys.has(plan.row.key))
for (const plan of retryPlans) {
try {
await processPlan(plan, true)
lastMediaKey = plan.row.key
} catch {
failedKeys.add(plan.row.key)
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
saveCheckpoint(message)
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
return
}
}
for (const plan of plans) {
if (plan.row.key <= lastMediaKey) continue
scanned += 1
try {
await processPlan(plan, false)
} catch {
failedKeys.add(plan.row.key)
lastMediaKey = plan.row.key
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
saveCheckpoint(message)
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
return
}
lastMediaKey = plan.row.key
const now = Date.now()
if (now - lastCheckpointAt >= 1000 || scanned === rows.length || failedKeys.size) {
saveCheckpoint(
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
lastCheckpointAt = now
}
}
if (failedKeys.size) {
markJobFailed(
jobId,
`${failedKeys.size} 条媒体同步失败`,
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
saveCheckpoint(
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
)
return
}
markJobDone(jobId, {
message: `扫描 ${rows.length}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
payload: {
targetBaseUrl: options.targetBaseUrl,
conflictMode: options.conflictMode,
direction: 'push',
lastMediaKey,
scanned: rows.length,
added,
skipped,
failed: 0,
filesSynced: totalFiles,
totalFiles,
failedKeys: [],
resumable: false
}
})
} catch (error) {
const message = error instanceof Error ? error.message : '两站点同步失败'
markJobFailed(jobId, message)
throw error
} finally {
tokens.delete(jobId)
}
}