站点同步
This commit is contained in:
1 parent
a64be50b73
commit
9cb8552a99
28 files changed
+1683
-18
No files matched your search
@@ -26,7 +26,8 @@ function typeLabel(type: BackupJobItem['type']): string {
|
|||||||
const map: Record<BackupJobItem['type'], string> = {
|
const map: Record<BackupJobItem['type'], string> = {
|
||||||
full_export: t('storage.backup.jobTypeExport'),
|
full_export: t('storage.backup.jobTypeExport'),
|
||||||
full_restore: t('storage.backup.jobTypeRestore'),
|
full_restore: t('storage.backup.jobTypeRestore'),
|
||||||
backend_sync: t('storage.backup.jobTypeSync')
|
backend_sync: t('storage.backup.jobTypeSync'),
|
||||||
|
site_sync: t('storage.backup.jobTypeSiteSync')
|
||||||
}
|
}
|
||||||
return map[type]
|
return map[type]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,256 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import type {
|
||||||
|
SiteSyncDirection,
|
||||||
|
SiteSyncRequest
|
||||||
|
} from '~/types/backup'
|
||||||
|
|
||||||
|
const props = defineProps<{
|
||||||
|
busy: boolean
|
||||||
|
disabled?: boolean
|
||||||
|
activeJob?: {
|
||||||
|
progress: number
|
||||||
|
total: number
|
||||||
|
message: string
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
} | null
|
||||||
|
resumableJob?: {
|
||||||
|
id: string
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
} | null
|
||||||
|
}>()
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
sync: [body: SiteSyncRequest]
|
||||||
|
}>()
|
||||||
|
|
||||||
|
const { t } = useI18n()
|
||||||
|
const targetBaseUrl = ref('')
|
||||||
|
const token = ref('')
|
||||||
|
const conflictMode = ref<'skip' | 'overwrite'>('skip')
|
||||||
|
const direction = ref<SiteSyncDirection>('push')
|
||||||
|
|
||||||
|
watch(() => props.resumableJob, (job) => {
|
||||||
|
if (!job) return
|
||||||
|
direction.value = job.payload.direction === 'pull' ? 'pull' : 'push'
|
||||||
|
if (typeof job.payload.targetBaseUrl === 'string') {
|
||||||
|
targetBaseUrl.value = job.payload.targetBaseUrl
|
||||||
|
}
|
||||||
|
}, { immediate: true })
|
||||||
|
|
||||||
|
const canSubmit = computed(() =>
|
||||||
|
!props.busy
|
||||||
|
&& !props.disabled
|
||||||
|
&& Boolean(targetBaseUrl.value.trim())
|
||||||
|
&& Boolean(token.value.trim())
|
||||||
|
)
|
||||||
|
const canResume = computed(() =>
|
||||||
|
!props.busy
|
||||||
|
&& !props.disabled
|
||||||
|
&& Boolean(props.resumableJob)
|
||||||
|
&& Boolean(token.value.trim())
|
||||||
|
)
|
||||||
|
const fileProgress = computed(() => {
|
||||||
|
const payload = props.activeJob?.payload
|
||||||
|
return {
|
||||||
|
synced: Number(payload?.filesSynced ?? 0),
|
||||||
|
total: Number(payload?.totalFiles ?? 0)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
const isPull = computed(() => direction.value === 'pull')
|
||||||
|
const directionSubtitle = computed(() => (
|
||||||
|
isPull.value
|
||||||
|
? t('storage.backup.siteSyncPullSubtitle')
|
||||||
|
: t('storage.backup.siteSyncPushSubtitle')
|
||||||
|
))
|
||||||
|
const endpointLabel = computed(() => (
|
||||||
|
isPull.value
|
||||||
|
? t('storage.backup.siteSyncSource')
|
||||||
|
: t('storage.backup.siteSyncTarget')
|
||||||
|
))
|
||||||
|
const tokenLabel = computed(() => (
|
||||||
|
isPull.value
|
||||||
|
? t('storage.backup.siteSyncSourceToken')
|
||||||
|
: t('storage.backup.siteSyncToken')
|
||||||
|
))
|
||||||
|
const tokenPlaceholder = computed(() => (
|
||||||
|
isPull.value
|
||||||
|
? t('storage.backup.siteSyncSourceTokenPlaceholder')
|
||||||
|
: t('storage.backup.siteSyncTokenPlaceholder')
|
||||||
|
))
|
||||||
|
const startLabel = computed(() => (
|
||||||
|
isPull.value
|
||||||
|
? t('storage.backup.siteSyncStartPull')
|
||||||
|
: t('storage.backup.siteSyncStartPush')
|
||||||
|
))
|
||||||
|
|
||||||
|
function submit(resume = false) {
|
||||||
|
if (resume ? !canResume.value : !canSubmit.value) return
|
||||||
|
emit('sync', {
|
||||||
|
targetBaseUrl: targetBaseUrl.value.trim(),
|
||||||
|
token: token.value.trim(),
|
||||||
|
conflictMode: conflictMode.value,
|
||||||
|
direction: direction.value,
|
||||||
|
...(resume && props.resumableJob ? { resumeJobId: props.resumableJob.id } : {})
|
||||||
|
})
|
||||||
|
token.value = ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function startNew() {
|
||||||
|
submit(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
function resumeSync() {
|
||||||
|
submit(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
function setConflictMode(mode: 'skip' | 'overwrite') {
|
||||||
|
conflictMode.value = mode
|
||||||
|
}
|
||||||
|
|
||||||
|
function setDirection(value: SiteSyncDirection) {
|
||||||
|
direction.value = value
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<article class="rounded-xl border border-default bg-elevated p-4 sm:p-5">
|
||||||
|
<div class="flex items-start gap-3">
|
||||||
|
<div class="flex size-10 shrink-0 items-center justify-center rounded-lg bg-primary/10 text-primary">
|
||||||
|
<UIcon
|
||||||
|
name="i-lucide-server-cog"
|
||||||
|
class="size-5"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="min-w-0">
|
||||||
|
<h3 class="text-base font-semibold">
|
||||||
|
{{ t('storage.backup.siteSyncTitle') }}
|
||||||
|
</h3>
|
||||||
|
<p class="mt-0.5 text-sm leading-relaxed text-muted">
|
||||||
|
{{ directionSubtitle }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-4 grid gap-3 lg:grid-cols-[minmax(0,1.2fr)_minmax(0,1fr)_auto] lg:items-end">
|
||||||
|
<div class="space-y-1.5">
|
||||||
|
<label class="text-xs text-muted">{{ endpointLabel }}</label>
|
||||||
|
<UInput
|
||||||
|
v-model="targetBaseUrl"
|
||||||
|
placeholder="https://target.example.com"
|
||||||
|
size="sm"
|
||||||
|
class="w-full"
|
||||||
|
:disabled="busy || disabled"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="space-y-1.5">
|
||||||
|
<label class="text-xs text-muted">{{ tokenLabel }}</label>
|
||||||
|
<UInput
|
||||||
|
v-model="token"
|
||||||
|
type="password"
|
||||||
|
:placeholder="tokenPlaceholder"
|
||||||
|
size="sm"
|
||||||
|
class="w-full"
|
||||||
|
:disabled="busy || disabled"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<UButton
|
||||||
|
:label="startLabel"
|
||||||
|
:icon="isPull ? 'i-lucide-cloud-download' : 'i-lucide-cloud-upload'"
|
||||||
|
color="primary"
|
||||||
|
:loading="busy"
|
||||||
|
:disabled="!canSubmit"
|
||||||
|
@click="startNew"
|
||||||
|
/>
|
||||||
|
<UButton
|
||||||
|
v-if="resumableJob"
|
||||||
|
:label="t('storage.backup.siteSyncResume')"
|
||||||
|
icon="i-lucide-play"
|
||||||
|
color="warning"
|
||||||
|
:loading="busy"
|
||||||
|
:disabled="!canResume"
|
||||||
|
class="lg:col-start-3"
|
||||||
|
@click="resumeSync"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-3 grid gap-3 sm:grid-cols-2">
|
||||||
|
<div class="flex flex-wrap items-center gap-2">
|
||||||
|
<span class="text-xs text-muted">{{ t('storage.backup.siteSyncDirection') }}</span>
|
||||||
|
<UButton
|
||||||
|
size="xs"
|
||||||
|
:variant="direction === 'push' ? 'solid' : 'ghost'"
|
||||||
|
:color="direction === 'push' ? 'primary' : 'neutral'"
|
||||||
|
:label="t('storage.backup.siteSyncPush')"
|
||||||
|
:disabled="busy || disabled"
|
||||||
|
@click="setDirection('push')"
|
||||||
|
/>
|
||||||
|
<UButton
|
||||||
|
size="xs"
|
||||||
|
:variant="direction === 'pull' ? 'solid' : 'ghost'"
|
||||||
|
:color="direction === 'pull' ? 'primary' : 'neutral'"
|
||||||
|
:label="t('storage.backup.siteSyncPull')"
|
||||||
|
:disabled="busy || disabled"
|
||||||
|
@click="setDirection('pull')"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="flex flex-wrap items-center gap-2 sm:justify-end">
|
||||||
|
<span class="text-xs text-muted">{{ t('storage.backup.siteSyncConflict') }}</span>
|
||||||
|
<UButton
|
||||||
|
size="xs"
|
||||||
|
:variant="conflictMode === 'skip' ? 'solid' : 'ghost'"
|
||||||
|
:color="conflictMode === 'skip' ? 'primary' : 'neutral'"
|
||||||
|
:label="t('storage.backup.siteSyncSkip')"
|
||||||
|
:disabled="busy || disabled"
|
||||||
|
@click="setConflictMode('skip')"
|
||||||
|
/>
|
||||||
|
<UButton
|
||||||
|
size="xs"
|
||||||
|
:variant="conflictMode === 'overwrite' ? 'solid' : 'ghost'"
|
||||||
|
:color="conflictMode === 'overwrite' ? 'primary' : 'neutral'"
|
||||||
|
:label="t('storage.backup.siteSyncOverwrite')"
|
||||||
|
:disabled="busy || disabled"
|
||||||
|
@click="setConflictMode('overwrite')"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<UAlert
|
||||||
|
v-if="activeJob"
|
||||||
|
class="mt-4"
|
||||||
|
color="primary"
|
||||||
|
variant="subtle"
|
||||||
|
icon="i-lucide-refresh-cw"
|
||||||
|
:description="activeJob.message"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-if="activeJob"
|
||||||
|
class="mt-3 grid gap-3 rounded-lg border border-default bg-default/40 p-3 sm:grid-cols-2"
|
||||||
|
>
|
||||||
|
<div>
|
||||||
|
<p class="text-xs text-muted">
|
||||||
|
{{ t('storage.backup.siteSyncScanProgress') }}
|
||||||
|
</p>
|
||||||
|
<p class="mt-1 font-mono text-sm font-medium">
|
||||||
|
{{ activeJob.message }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p class="text-xs text-muted">
|
||||||
|
{{ t('storage.backup.siteSyncFileProgress') }}
|
||||||
|
</p>
|
||||||
|
<p class="mt-1 font-mono text-sm font-medium">
|
||||||
|
{{ fileProgress.synced }} / {{ fileProgress.total }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<BackupJobProgress
|
||||||
|
v-if="activeJob"
|
||||||
|
class="mt-2"
|
||||||
|
:message="activeJob.message"
|
||||||
|
:progress="activeJob.progress"
|
||||||
|
:total="activeJob.total"
|
||||||
|
/>
|
||||||
|
</article>
|
||||||
|
</template>
|
||||||
@@ -1,5 +1,9 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import type { BackupJobItem, SyncDraft } from '~/types/backup'
|
import type {
|
||||||
|
BackupJobItem,
|
||||||
|
SiteSyncRequest,
|
||||||
|
SyncDraft
|
||||||
|
} from '~/types/backup'
|
||||||
import type { StorageBackendItem } from '~/types/storage'
|
import type { StorageBackendItem } from '~/types/storage'
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
@@ -31,6 +35,7 @@ const {
|
|||||||
confirmRestore,
|
confirmRestore,
|
||||||
previewSync,
|
previewSync,
|
||||||
startSync,
|
startSync,
|
||||||
|
startSiteSync,
|
||||||
retryJob,
|
retryJob,
|
||||||
cancelJob,
|
cancelJob,
|
||||||
downloadUrl
|
downloadUrl
|
||||||
@@ -45,6 +50,14 @@ const activeJob = computed(() => getActiveJob())
|
|||||||
const exportBusy = computed(() => activeJob.value?.type === 'full_export')
|
const exportBusy = computed(() => activeJob.value?.type === 'full_export')
|
||||||
const restoreBusy = computed(() => activeJob.value?.type === 'full_restore')
|
const restoreBusy = computed(() => activeJob.value?.type === 'full_restore')
|
||||||
const syncBusy = computed(() => activeJob.value?.type === 'backend_sync')
|
const syncBusy = computed(() => activeJob.value?.type === 'backend_sync')
|
||||||
|
const siteSyncBusy = computed(() => activeJob.value?.type === 'site_sync')
|
||||||
|
const resumableSiteSyncJob = computed(() =>
|
||||||
|
jobs.value.find(job =>
|
||||||
|
job.type === 'site_sync'
|
||||||
|
&& job.status === 'failed'
|
||||||
|
&& job.payload.resumable === true
|
||||||
|
) ?? null
|
||||||
|
)
|
||||||
const anyJobBusy = computed(() => Boolean(activeJob.value))
|
const anyJobBusy = computed(() => Boolean(activeJob.value))
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
@@ -94,6 +107,10 @@ async function onSync(body: {
|
|||||||
await startSync(body)
|
await startSync(body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function onSiteSync(body: SiteSyncRequest) {
|
||||||
|
await startSiteSync(body)
|
||||||
|
}
|
||||||
|
|
||||||
function onDownload(jobId: string) {
|
function onDownload(jobId: string) {
|
||||||
window.open(downloadUrl(jobId), '_blank')
|
window.open(downloadUrl(jobId), '_blank')
|
||||||
}
|
}
|
||||||
@@ -159,6 +176,14 @@ function onViewJob(job: BackupJobItem) {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<StorageSiteSyncCard
|
||||||
|
:busy="siteSyncBusy"
|
||||||
|
:disabled="anyJobBusy && !siteSyncBusy"
|
||||||
|
:active-job="siteSyncBusy ? activeJob : null"
|
||||||
|
:resumable-job="resumableSiteSyncJob"
|
||||||
|
@sync="onSiteSync"
|
||||||
|
/>
|
||||||
|
|
||||||
<StorageBackupJobTable
|
<StorageBackupJobTable
|
||||||
:jobs="jobs"
|
:jobs="jobs"
|
||||||
:polling="polling"
|
:polling="polling"
|
||||||
|
|||||||
@@ -3,7 +3,8 @@ import type {
|
|||||||
BackupJobItem,
|
BackupJobItem,
|
||||||
BackupPackageItem,
|
BackupPackageItem,
|
||||||
BackupRestorePreview,
|
BackupRestorePreview,
|
||||||
BackendSyncPreview
|
BackendSyncPreview,
|
||||||
|
SiteSyncRequest
|
||||||
} from '~/types/backup'
|
} from '~/types/backup'
|
||||||
|
|
||||||
const jobs = ref<BackupJobItem[]>([])
|
const jobs = ref<BackupJobItem[]>([])
|
||||||
@@ -255,6 +256,26 @@ export function useBackup() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function startSiteSync(body: SiteSyncRequest) {
|
||||||
|
try {
|
||||||
|
const data = await $fetch<{ job: BackupJobItem }>(
|
||||||
|
'/api/admin/site-sync',
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
credentials: 'include',
|
||||||
|
body
|
||||||
|
}
|
||||||
|
)
|
||||||
|
await refreshJobsFirstPage()
|
||||||
|
startPolling()
|
||||||
|
toast.add({ title: t('storage.backup.siteSyncStarted'), color: 'success' })
|
||||||
|
return data.job
|
||||||
|
} catch (error: unknown) {
|
||||||
|
handleError(error, t('storage.backup.siteSyncFailed'))
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function retryJob(jobId: string) {
|
async function retryJob(jobId: string) {
|
||||||
try {
|
try {
|
||||||
await $fetch(`/api/admin/backup/jobs/${jobId}/retry`, {
|
await $fetch(`/api/admin/backup/jobs/${jobId}/retry`, {
|
||||||
@@ -312,6 +333,7 @@ export function useBackup() {
|
|||||||
confirmRestore,
|
confirmRestore,
|
||||||
previewSync,
|
previewSync,
|
||||||
startSync,
|
startSync,
|
||||||
|
startSiteSync,
|
||||||
retryJob,
|
retryJob,
|
||||||
cancelJob,
|
cancelJob,
|
||||||
downloadUrl
|
downloadUrl
|
||||||
|
|||||||
+10
-1
@@ -1,5 +1,6 @@
|
|||||||
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync'
|
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
|
||||||
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
|
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
|
||||||
|
export type SiteSyncDirection = 'push' | 'pull'
|
||||||
|
|
||||||
export interface BackupJobItem {
|
export interface BackupJobItem {
|
||||||
id: string
|
id: string
|
||||||
@@ -51,3 +52,11 @@ export interface SyncDraft {
|
|||||||
fromBackendId: string
|
fromBackendId: string
|
||||||
toBackendId: string
|
toBackendId: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SiteSyncRequest {
|
||||||
|
targetBaseUrl: string
|
||||||
|
token: string
|
||||||
|
conflictMode: 'skip' | 'overwrite'
|
||||||
|
direction: SiteSyncDirection
|
||||||
|
resumeJobId?: string
|
||||||
|
}
|
||||||
@@ -645,6 +645,30 @@
|
|||||||
"conflictSkip": "Skip existing files",
|
"conflictSkip": "Skip existing files",
|
||||||
"conflictOverwrite": "Overwrite existing files and database",
|
"conflictOverwrite": "Overwrite existing files and database",
|
||||||
"syncTitle": "Cross-backend sync",
|
"syncTitle": "Cross-backend sync",
|
||||||
|
"siteSyncTitle": "Site-to-site sync",
|
||||||
|
"siteSyncSubtitle": "Sync media indexes, images, videos, files, thumbnails, HLS segments and tags to another PicHost. Matching usernames must already exist on the target.",
|
||||||
|
"siteSyncPushSubtitle": "Push media indexes, images, videos, files, thumbnails, HLS segments and tags to another PicHost. Matching usernames must already exist on the target.",
|
||||||
|
"siteSyncPullSubtitle": "Pull media indexes, images, videos, files, thumbnails, HLS segments and tags from another PicHost. Matching usernames must already exist on this site.",
|
||||||
|
"siteSyncTarget": "Target site URL",
|
||||||
|
"siteSyncSource": "Source site URL",
|
||||||
|
"siteSyncToken": "Target admin API token",
|
||||||
|
"siteSyncSourceToken": "Source admin API token",
|
||||||
|
"siteSyncTokenPlaceholder": "Enter target admin token",
|
||||||
|
"siteSyncSourceTokenPlaceholder": "Enter source admin token",
|
||||||
|
"siteSyncStart": "Start sync",
|
||||||
|
"siteSyncStartPush": "Start push",
|
||||||
|
"siteSyncStartPull": "Start pull",
|
||||||
|
"siteSyncDirection": "Sync direction:",
|
||||||
|
"siteSyncPush": "Push to target",
|
||||||
|
"siteSyncPull": "Pull from target",
|
||||||
|
"siteSyncConflict": "Conflict policy:",
|
||||||
|
"siteSyncSkip": "Skip existing",
|
||||||
|
"siteSyncOverwrite": "Overwrite existing",
|
||||||
|
"siteSyncResume": "Resume last sync",
|
||||||
|
"siteSyncScanProgress": "Media scan progress",
|
||||||
|
"siteSyncFileProgress": "Synced file count",
|
||||||
|
"siteSyncStarted": "Site-to-site sync started",
|
||||||
|
"siteSyncFailed": "Failed to start site-to-site sync",
|
||||||
"syncMigrateTitle": "Cross-backend migration",
|
"syncMigrateTitle": "Cross-backend migration",
|
||||||
"syncMigrateSubtitle": "Migrate files between configured backends",
|
"syncMigrateSubtitle": "Migrate files between configured backends",
|
||||||
"syncSubtitle": "Copy files between configured backends and update the index",
|
"syncSubtitle": "Copy files between configured backends and update the index",
|
||||||
@@ -694,6 +718,7 @@
|
|||||||
"jobTypeExport": "Site export",
|
"jobTypeExport": "Site export",
|
||||||
"jobTypeRestore": "Site restore",
|
"jobTypeRestore": "Site restore",
|
||||||
"jobTypeSync": "Cross-backend sync",
|
"jobTypeSync": "Cross-backend sync",
|
||||||
|
"jobTypeSiteSync": "Site-to-site sync",
|
||||||
"statusPending": "Pending",
|
"statusPending": "Pending",
|
||||||
"statusRunning": "Running",
|
"statusRunning": "Running",
|
||||||
"statusDone": "Done",
|
"statusDone": "Done",
|
||||||
|
|||||||
@@ -645,6 +645,30 @@
|
|||||||
"conflictSkip": "跳过已存在文件",
|
"conflictSkip": "跳过已存在文件",
|
||||||
"conflictOverwrite": "覆盖已存在文件与数据库",
|
"conflictOverwrite": "覆盖已存在文件与数据库",
|
||||||
"syncTitle": "跨后端同步",
|
"syncTitle": "跨后端同步",
|
||||||
|
"siteSyncTitle": "两站点同步",
|
||||||
|
"siteSyncSubtitle": "将当前站点的媒体索引、图片、视频、文件、缩略图、HLS 分片和标签同步到另一台 PicHost。目标站点需已有同名用户。",
|
||||||
|
"siteSyncPushSubtitle": "将当前站点的媒体索引、图片、视频、文件、缩略图、HLS 分片和标签推送到另一台 PicHost。目标站点需已有同名用户。",
|
||||||
|
"siteSyncPullSubtitle": "从另一台 PicHost 拉取媒体索引、图片、视频、文件、缩略图、HLS 分片和标签。当前站点需已有同名用户。",
|
||||||
|
"siteSyncTarget": "目标站点地址",
|
||||||
|
"siteSyncSource": "来源站点地址",
|
||||||
|
"siteSyncToken": "目标站点管理员 API Token",
|
||||||
|
"siteSyncSourceToken": "来源站点管理员 API Token",
|
||||||
|
"siteSyncTokenPlaceholder": "输入目标站点管理员 Token",
|
||||||
|
"siteSyncSourceTokenPlaceholder": "输入来源站点管理员 Token",
|
||||||
|
"siteSyncStart": "开始同步",
|
||||||
|
"siteSyncStartPush": "开始推送",
|
||||||
|
"siteSyncStartPull": "开始拉取",
|
||||||
|
"siteSyncDirection": "同步策略:",
|
||||||
|
"siteSyncPush": "推送到目标站点",
|
||||||
|
"siteSyncPull": "从目标站点拉取",
|
||||||
|
"siteSyncConflict": "冲突策略:",
|
||||||
|
"siteSyncSkip": "跳过已存在",
|
||||||
|
"siteSyncOverwrite": "覆盖已存在",
|
||||||
|
"siteSyncResume": "继续上次同步",
|
||||||
|
"siteSyncScanProgress": "媒体扫描进度",
|
||||||
|
"siteSyncFileProgress": "已同步文件数量",
|
||||||
|
"siteSyncStarted": "两站点同步任务已开始",
|
||||||
|
"siteSyncFailed": "启动两站点同步失败",
|
||||||
"syncMigrateTitle": "跨后端迁移",
|
"syncMigrateTitle": "跨后端迁移",
|
||||||
"syncMigrateSubtitle": "在已配置后端之间迁移文件",
|
"syncMigrateSubtitle": "在已配置后端之间迁移文件",
|
||||||
"syncSubtitle": "在已配置的后端之间拷贝文件并更新索引",
|
"syncSubtitle": "在已配置的后端之间拷贝文件并更新索引",
|
||||||
@@ -694,6 +718,7 @@
|
|||||||
"jobTypeExport": "站点导出",
|
"jobTypeExport": "站点导出",
|
||||||
"jobTypeRestore": "站点恢复",
|
"jobTypeRestore": "站点恢复",
|
||||||
"jobTypeSync": "跨后端同步",
|
"jobTypeSync": "跨后端同步",
|
||||||
|
"jobTypeSiteSync": "两站点同步",
|
||||||
"statusPending": "等待中",
|
"statusPending": "等待中",
|
||||||
"statusRunning": "进行中",
|
"statusRunning": "进行中",
|
||||||
"statusDone": "已完成",
|
"statusDone": "已完成",
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import { requireAdminAuth } from '../../utils/access'
|
||||||
|
import { createApiError } from '../../utils/api-error'
|
||||||
|
import { serializeBackupJob } from '../../utils/backup/api'
|
||||||
|
import {
|
||||||
|
resumeSiteSyncJob,
|
||||||
|
startSiteSyncJob
|
||||||
|
} from '../../utils/backup/runner'
|
||||||
|
|
||||||
|
interface SiteSyncBody {
|
||||||
|
targetBaseUrl?: string
|
||||||
|
token?: string
|
||||||
|
conflictMode?: 'skip' | 'overwrite'
|
||||||
|
direction?: 'push' | 'pull'
|
||||||
|
resumeJobId?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
const user = await requireAdminAuth(event)
|
||||||
|
const body = await readBody<SiteSyncBody>(event).catch(() => null)
|
||||||
|
const targetBaseUrl = body?.targetBaseUrl?.trim() ?? ''
|
||||||
|
const token = body?.token?.trim() ?? ''
|
||||||
|
const conflictMode = body?.conflictMode === 'skip' ? 'skip' : 'overwrite'
|
||||||
|
const direction = body?.direction === 'pull' ? 'pull' : 'push'
|
||||||
|
|
||||||
|
if (!token) {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', '请输入目标站点管理员 API Token', 400)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
let job
|
||||||
|
if (body?.resumeJobId) {
|
||||||
|
job = resumeSiteSyncJob(body.resumeJobId, token, user.id)
|
||||||
|
} else {
|
||||||
|
let targetUrl: URL
|
||||||
|
try {
|
||||||
|
targetUrl = new URL(targetBaseUrl)
|
||||||
|
if (targetUrl.protocol !== 'http:' && targetUrl.protocol !== 'https:') {
|
||||||
|
throw new Error('invalid protocol')
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', '请输入有效的目标站点地址', 400)
|
||||||
|
}
|
||||||
|
job = startSiteSyncJob({
|
||||||
|
targetBaseUrl: targetUrl.toString(),
|
||||||
|
conflictMode,
|
||||||
|
direction
|
||||||
|
}, token, user.id)
|
||||||
|
}
|
||||||
|
return { job: serializeBackupJob(job) }
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Error && error.message === 'BACKUP_JOB_RUNNING') {
|
||||||
|
createApiError(event, 'CONFLICT', '已有备份或迁移任务正在运行', 409)
|
||||||
|
}
|
||||||
|
if (error instanceof Error && error.message === 'SITE_SYNC_JOB_NOT_RESUMABLE') {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', '该同步任务无法继续', 400)
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -3,7 +3,7 @@ import { assertImageOwnership, requireApiOrAdminAuth } from '../../utils/access'
|
|||||||
import { createApiError } from '../../utils/api-error'
|
import { createApiError } from '../../utils/api-error'
|
||||||
import { logActivity } from '../../utils/activity-log'
|
import { logActivity } from '../../utils/activity-log'
|
||||||
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
|
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
|
||||||
import { validateFileKey, validateVideoKey } from '../../utils/image-key'
|
import { validateStorageKey, validateVideoKey } from '../../utils/image-key'
|
||||||
import { deleteImage, headImage } from '../../utils/storage'
|
import { deleteImage, headImage } from '../../utils/storage'
|
||||||
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
|
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
|
||||||
import { MAX_DELETE_BATCH } from '../../utils/constants'
|
import { MAX_DELETE_BATCH } from '../../utils/constants'
|
||||||
@@ -31,7 +31,7 @@ export default defineEventHandler(async (event) => {
|
|||||||
const failed: BatchDeleteResponse['failed'] = []
|
const failed: BatchDeleteResponse['failed'] = []
|
||||||
|
|
||||||
for (const key of keys) {
|
for (const key of keys) {
|
||||||
if (!validateFileKey(key) && !validateVideoKey(key)) {
|
if (!validateStorageKey(key)) {
|
||||||
failed.push({ key, error: { code: 'INVALID_IMAGE_KEY', message: '无效的文件路径' } })
|
failed.push({ key, error: { code: 'INVALID_IMAGE_KEY', message: '无效的文件路径' } })
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { assertImageOwnership, requireApiOrAdminAuth } from '../../utils/access'
|
|||||||
import { createApiError } from '../../utils/api-error'
|
import { createApiError } from '../../utils/api-error'
|
||||||
import { logActivity } from '../../utils/activity-log'
|
import { logActivity } from '../../utils/activity-log'
|
||||||
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
|
import { deleteImageIndex, getImageIndexRow } from '../../utils/image-index'
|
||||||
import { validateFileKey, validateVideoKey } from '../../utils/image-key'
|
import { validateStorageKey, validateVideoKey } from '../../utils/image-key'
|
||||||
import { deleteImage, headImage } from '../../utils/storage'
|
import { deleteImage, headImage } from '../../utils/storage'
|
||||||
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
|
import { cancelTranscode, removeHlsArtifacts } from '../../utils/transcode'
|
||||||
import { logInfo } from '../../utils/logger'
|
import { logInfo } from '../../utils/logger'
|
||||||
@@ -20,7 +20,7 @@ export default defineEventHandler(async (event) => {
|
|||||||
const body = await readBody<DeleteBody>(event).catch(() => null)
|
const body = await readBody<DeleteBody>(event).catch(() => null)
|
||||||
const key = (typeof query.key === 'string' ? query.key : undefined) ?? body?.key
|
const key = (typeof query.key === 'string' ? query.key : undefined) ?? body?.key
|
||||||
|
|
||||||
if (!key || (!validateFileKey(key) && !validateVideoKey(key))) {
|
if (!key || !validateStorageKey(key)) {
|
||||||
createApiError(event, 'INVALID_IMAGE_KEY', '无效的文件路径', 400)
|
createApiError(event, 'INVALID_IMAGE_KEY', '无效的文件路径', 400)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
import { requireSiteSyncAuth } from '../../utils/access'
|
||||||
|
import { listAllImageIndexRows } from '../../utils/image-index'
|
||||||
|
import { listUserIdUsernameMap } from '../../utils/db'
|
||||||
|
import { collectMediaAssetKeys } from '../../utils/media-assets'
|
||||||
|
import { getBackendForKey } from '../../utils/storage/resolver'
|
||||||
|
import type { SiteSyncInventory } from '../../utils/site-sync/protocol'
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event): Promise<SiteSyncInventory> => {
|
||||||
|
await requireSiteSyncAuth(event)
|
||||||
|
|
||||||
|
const rows = listAllImageIndexRows()
|
||||||
|
const objects = new Map<string, number>()
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const backend = await getBackendForKey(row.key)
|
||||||
|
let keys: string[]
|
||||||
|
try {
|
||||||
|
keys = await collectMediaAssetKeys(row, backend)
|
||||||
|
} catch {
|
||||||
|
keys = [
|
||||||
|
...(Number(row.source_deleted ?? 0) !== 1 ? [row.key] : []),
|
||||||
|
...(row.thumb_key ? [row.thumb_key] : []),
|
||||||
|
...(row.hls_key ? [row.hls_key] : [])
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const key of new Set(keys)) {
|
||||||
|
const stored = await backend.head(key)
|
||||||
|
if (stored) objects.set(key, stored.size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
users: [...listUserIdUsernameMap().values()].sort(),
|
||||||
|
mediaKeys: rows.map(row => row.key),
|
||||||
|
objects: [...objects].map(([key, size]) => ({ key, size }))
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { getQuery } from 'h3'
|
||||||
|
import { requireSiteSyncAuth } from '../../utils/access'
|
||||||
|
import { createApiError } from '../../utils/api-error'
|
||||||
|
import { listUserIdUsernameMap } from '../../utils/db'
|
||||||
|
import {
|
||||||
|
getImageIndexRow,
|
||||||
|
getImageTotals,
|
||||||
|
listImageIndexRowsAfter
|
||||||
|
} from '../../utils/image-index'
|
||||||
|
import { validateStorageKey } from '../../utils/image-key'
|
||||||
|
import { collectMediaAssetKeys } from '../../utils/media-assets'
|
||||||
|
import { getBackendForKey } from '../../utils/storage/resolver'
|
||||||
|
import { getTagsForImageKeys } from '../../utils/tags'
|
||||||
|
import { toSiteSyncMediaTransfer } from '../../utils/site-sync/manifest'
|
||||||
|
import type {
|
||||||
|
SiteSyncMediaAsset,
|
||||||
|
SiteSyncMediaBundle,
|
||||||
|
SiteSyncMediaPage
|
||||||
|
} from '../../utils/site-sync/protocol'
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event): Promise<SiteSyncMediaPage> => {
|
||||||
|
await requireSiteSyncAuth(event)
|
||||||
|
|
||||||
|
const query = getQuery(event)
|
||||||
|
const key = typeof query.key === 'string' ? query.key.trim() : ''
|
||||||
|
const after = typeof query.after === 'string' ? query.after : ''
|
||||||
|
const limit = Math.min(
|
||||||
|
100,
|
||||||
|
Math.max(1, Number.parseInt(String(query.limit ?? '50'), 10) || 50)
|
||||||
|
)
|
||||||
|
|
||||||
|
if (key && !validateStorageKey(key)) {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', '媒体 key 无效', 400)
|
||||||
|
}
|
||||||
|
|
||||||
|
const rows = key
|
||||||
|
? [getImageIndexRow(key)].filter(row => row !== null)
|
||||||
|
: listImageIndexRowsAfter(after, limit)
|
||||||
|
if (key && rows.length === 0) {
|
||||||
|
createApiError(event, 'IMAGE_NOT_FOUND', '目标媒体不存在', 404)
|
||||||
|
}
|
||||||
|
|
||||||
|
const ownerMap = listUserIdUsernameMap()
|
||||||
|
const tagMap = getTagsForImageKeys(rows.map(row => row.key))
|
||||||
|
const items: SiteSyncMediaBundle[] = []
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const backend = await getBackendForKey(row.key)
|
||||||
|
const keys = await collectMediaAssetKeys(row, backend)
|
||||||
|
const assets: SiteSyncMediaAsset[] = []
|
||||||
|
for (const assetKey of new Set(keys)) {
|
||||||
|
const stored = await backend.head(assetKey)
|
||||||
|
if (!stored) {
|
||||||
|
throw new Error(`目标媒体对象不存在:${assetKey}`)
|
||||||
|
}
|
||||||
|
assets.push({ key: assetKey, size: stored.size })
|
||||||
|
}
|
||||||
|
items.push({
|
||||||
|
media: toSiteSyncMediaTransfer(
|
||||||
|
row,
|
||||||
|
row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
|
||||||
|
tagMap.get(row.key) ?? []
|
||||||
|
),
|
||||||
|
assets
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
items,
|
||||||
|
totalMedia: getImageTotals().count,
|
||||||
|
nextCursor: key || rows.length < limit
|
||||||
|
? null
|
||||||
|
: rows.at(-1)?.key ?? null
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import { requireSiteSyncAuth } from '../../utils/access'
|
||||||
|
import { createApiError } from '../../utils/api-error'
|
||||||
|
import { validateStorageKey } from '../../utils/image-key'
|
||||||
|
import { applySiteSyncMedia } from '../../utils/site-sync/apply'
|
||||||
|
import type { SiteSyncMediaTransfer } from '../../utils/site-sync/protocol'
|
||||||
|
|
||||||
|
interface Body {
|
||||||
|
media?: SiteSyncMediaTransfer
|
||||||
|
conflictMode?: 'skip' | 'overwrite'
|
||||||
|
}
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
const admin = await requireSiteSyncAuth(event)
|
||||||
|
const body = await readBody<Body>(event).catch(() => null)
|
||||||
|
const media = body?.media
|
||||||
|
if (!media || !validateStorageKey(media.key)) {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', '媒体同步参数无效', 400)
|
||||||
|
}
|
||||||
|
const conflictMode = body?.conflictMode === 'skip' ? 'skip' : 'overwrite'
|
||||||
|
try {
|
||||||
|
return await applySiteSyncMedia(media, conflictMode, admin.id)
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : '媒体同步失败'
|
||||||
|
createApiError(event, 'INVALID_REQUEST', message, 400)
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import { getQuery, sendStream, setResponseHeader } from 'h3'
|
||||||
|
import { requireSiteSyncAuth } from '../../utils/access'
|
||||||
|
import { createApiError } from '../../utils/api-error'
|
||||||
|
import { validateStorageKey } from '../../utils/image-key'
|
||||||
|
import { getBackendForKey } from '../../utils/storage/resolver'
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
await requireSiteSyncAuth(event)
|
||||||
|
|
||||||
|
const query = getQuery(event)
|
||||||
|
const key = typeof query.key === 'string' ? query.key.trim() : ''
|
||||||
|
if (!validateStorageKey(key)) {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', '同步对象 key 无效', 400)
|
||||||
|
}
|
||||||
|
|
||||||
|
const backend = await getBackendForKey(key)
|
||||||
|
const stored = await backend.head(key)
|
||||||
|
if (!stored) {
|
||||||
|
createApiError(event, 'IMAGE_NOT_FOUND', `目标对象不存在:${key}`, 404)
|
||||||
|
}
|
||||||
|
|
||||||
|
setResponseHeader(event, 'Content-Type', stored.contentType)
|
||||||
|
setResponseHeader(event, 'Content-Length', stored.size)
|
||||||
|
setResponseHeader(event, 'Cache-Control', 'no-store')
|
||||||
|
return sendStream(event, await backend.createStream(key))
|
||||||
|
})
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
import { readMultipartFormData } from 'h3'
|
||||||
|
import { requireSiteSyncAuth } from '../../utils/access'
|
||||||
|
import { createApiError } from '../../utils/api-error'
|
||||||
|
import { MAX_VIDEO_FILE_SIZE } from '../../utils/constants'
|
||||||
|
import { findUserByUsername } from '../../utils/db'
|
||||||
|
import { validateStorageKey } from '../../utils/image-key'
|
||||||
|
import { getActiveBackend } from '../../utils/storage/resolver'
|
||||||
|
|
||||||
|
function field(form: Awaited<ReturnType<typeof readMultipartFormData>>, name: string): string {
|
||||||
|
const part = form?.find(item => item.name === name)
|
||||||
|
return part?.data?.length ? new TextDecoder().decode(part.data).trim() : ''
|
||||||
|
}
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
await requireSiteSyncAuth(event)
|
||||||
|
const form = await readMultipartFormData(event)
|
||||||
|
if (!form?.length) createApiError(event, 'INVALID_REQUEST', '未收到同步数据', 400)
|
||||||
|
|
||||||
|
const key = field(form, 'key')
|
||||||
|
const file = form.find(part => part.name === 'file' && part.data?.length)
|
||||||
|
if (!validateStorageKey(key) || !file?.data) {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', '同步文件参数无效', 400)
|
||||||
|
}
|
||||||
|
if (file.data.length > MAX_VIDEO_FILE_SIZE) {
|
||||||
|
createApiError(event, 'FILE_TOO_LARGE', '同步文件超过 500 MB', 413)
|
||||||
|
}
|
||||||
|
|
||||||
|
const conflictMode = field(form, 'conflictMode') === 'skip' ? 'skip' : 'overwrite'
|
||||||
|
const backend = await getActiveBackend()
|
||||||
|
if (conflictMode === 'skip' && await backend.head(key)) {
|
||||||
|
return { success: true, skipped: true }
|
||||||
|
}
|
||||||
|
|
||||||
|
const ownerUsername = field(form, 'ownerUsername')
|
||||||
|
const owner = ownerUsername ? findUserByUsername(ownerUsername) : null
|
||||||
|
if (ownerUsername && !owner) {
|
||||||
|
createApiError(event, 'INVALID_REQUEST', `目标站点缺少用户:${ownerUsername}`, 400)
|
||||||
|
}
|
||||||
|
|
||||||
|
await backend.put(key, new Uint8Array(file.data), {
|
||||||
|
originalName: field(form, 'originalName') || file.filename || key.split('/').pop() || 'file',
|
||||||
|
uploadedAt: field(form, 'uploadedAt') || new Date().toISOString(),
|
||||||
|
contentType: file.type || field(form, 'contentType') || 'application/octet-stream',
|
||||||
|
size: file.data.length,
|
||||||
|
userId: owner?.id ?? null,
|
||||||
|
mediaType: key.startsWith('videos/')
|
||||||
|
? 'video'
|
||||||
|
: key.startsWith('files/')
|
||||||
|
? 'file'
|
||||||
|
: 'image'
|
||||||
|
})
|
||||||
|
|
||||||
|
return { success: true, skipped: false }
|
||||||
|
})
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import { requireSiteSyncAuth } from '../../utils/access'
|
||||||
|
import { listUserIdUsernameMap } from '../../utils/db'
|
||||||
|
import type { SiteSyncUsers } from '../../utils/site-sync/protocol'
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event): Promise<SiteSyncUsers> => {
|
||||||
|
await requireSiteSyncAuth(event)
|
||||||
|
return {
|
||||||
|
users: [...listUserIdUsernameMap().values()].sort()
|
||||||
|
}
|
||||||
|
})
|
||||||
+31
-1
@@ -1,5 +1,5 @@
|
|||||||
import type { H3Event } from 'h3'
|
import type { H3Event } from 'h3'
|
||||||
import { findUserByApiToken, getPrimaryAdminUserId } from './db'
|
import { findUserById, findUserByApiToken, getPrimaryAdminUserId } from './db'
|
||||||
import { getAdminSecret, getApiUploadToken } from './env'
|
import { getAdminSecret, getApiUploadToken } from './env'
|
||||||
import { createApiError } from './api-error'
|
import { createApiError } from './api-error'
|
||||||
import {
|
import {
|
||||||
@@ -110,6 +110,36 @@ export function resolveUserIdFromApiToken(
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 两站点同步:全站 Token 或管理员个人 Token;也允许管理员网页会话。 */
|
||||||
|
export async function requireSiteSyncAuth(event: H3Event): Promise<AuthUser> {
|
||||||
|
if (devBypass(event)) {
|
||||||
|
return { id: 0, username: 'dev', role: 'admin' }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (verifyGlobalApiUploadToken(event)) {
|
||||||
|
const adminId = getPrimaryAdminUserId()
|
||||||
|
if (adminId != null) {
|
||||||
|
const admin = findUserById(adminId)
|
||||||
|
if (admin) {
|
||||||
|
return { id: admin.id, username: admin.username, role: admin.role }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const userId = resolveUserIdFromApiToken(event)
|
||||||
|
if (userId != null) {
|
||||||
|
const user = findUserById(userId)
|
||||||
|
if (user?.role === 'admin') {
|
||||||
|
return { id: user.id, username: user.username, role: user.role }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionUser = await getCurrentUser(event)
|
||||||
|
if (sessionUser?.role === 'admin') return sessionUser
|
||||||
|
|
||||||
|
createApiError(event, 'FORBIDDEN', '需要目标站点管理员同步 Token', 403)
|
||||||
|
}
|
||||||
|
|
||||||
export function verifyApiUploadToken(event: H3Event): boolean {
|
export function verifyApiUploadToken(event: H3Event): boolean {
|
||||||
return verifyGlobalApiUploadToken(event)
|
return verifyGlobalApiUploadToken(event)
|
||||||
|| resolveUserIdFromApiToken(event) !== null
|
|| resolveUserIdFromApiToken(event) !== null
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ export {
|
|||||||
startFullExportJob,
|
startFullExportJob,
|
||||||
startFullRestoreJob,
|
startFullRestoreJob,
|
||||||
startBackendSyncJob,
|
startBackendSyncJob,
|
||||||
|
startSiteSyncJob,
|
||||||
|
resumeSiteSyncJob,
|
||||||
isBackupRunnerBusy,
|
isBackupRunnerBusy,
|
||||||
cancelBackupJob
|
cancelBackupJob
|
||||||
} from './runner'
|
} from './runner'
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { randomBytes } from 'node:crypto'
|
|||||||
import { getDb, getPrimaryAdminUserId } from '../db'
|
import { getDb, getPrimaryAdminUserId } from '../db'
|
||||||
import { ensureBackupSchema } from './schema'
|
import { ensureBackupSchema } from './schema'
|
||||||
|
|
||||||
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync'
|
export type BackupJobType = 'full_export' | 'full_restore' | 'backend_sync' | 'site_sync'
|
||||||
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
|
export type BackupJobStatus = 'pending' | 'running' | 'done' | 'failed'
|
||||||
|
|
||||||
export interface BackupJob {
|
export interface BackupJob {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { runBackendSyncJob } from './backend-sync'
|
import { runBackendSyncJob } from './backend-sync'
|
||||||
import { runFullExportJob } from './full-export'
|
import { runFullExportJob } from './full-export'
|
||||||
import { runFullRestoreJob } from './full-restore'
|
import { runFullRestoreJob } from './full-restore'
|
||||||
|
import { registerSiteSyncToken, runSiteSyncJob, type SiteSyncOptions } from '../site-sync/runner'
|
||||||
import {
|
import {
|
||||||
createBackupJob,
|
createBackupJob,
|
||||||
getBackupJob,
|
getBackupJob,
|
||||||
@@ -114,3 +115,55 @@ export function startBackendSyncJob(options: {
|
|||||||
scheduleBackupJob(job, () => runBackendSyncJob(job.id, options))
|
scheduleBackupJob(job, () => runBackendSyncJob(job.id, options))
|
||||||
return getBackupJob(job.id)!
|
return getBackupJob(job.id)!
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function startSiteSyncJob(
|
||||||
|
options: SiteSyncOptions,
|
||||||
|
token: string,
|
||||||
|
userId?: number | null
|
||||||
|
): BackupJob {
|
||||||
|
claimJobSlot()
|
||||||
|
const job = createBackupJob({
|
||||||
|
type: 'site_sync',
|
||||||
|
userId,
|
||||||
|
message: '等待开始…',
|
||||||
|
payload: {
|
||||||
|
targetBaseUrl: options.targetBaseUrl,
|
||||||
|
conflictMode: options.conflictMode,
|
||||||
|
direction: options.direction
|
||||||
|
}
|
||||||
|
})
|
||||||
|
registerSiteSyncToken(job.id, token)
|
||||||
|
scheduleBackupJob(job, () => runSiteSyncJob(job.id, options))
|
||||||
|
return getBackupJob(job.id)!
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resumeSiteSyncJob(
|
||||||
|
jobId: string,
|
||||||
|
token: string,
|
||||||
|
userId?: number | null
|
||||||
|
): BackupJob {
|
||||||
|
claimJobSlot()
|
||||||
|
const job = getBackupJob(jobId, userId ?? undefined)
|
||||||
|
if (
|
||||||
|
!job
|
||||||
|
|| job.type !== 'site_sync'
|
||||||
|
|| job.status !== 'failed'
|
||||||
|
|| job.payload.resumable !== true
|
||||||
|
) {
|
||||||
|
throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
||||||
|
}
|
||||||
|
const targetBaseUrl = typeof job.payload.targetBaseUrl === 'string'
|
||||||
|
? job.payload.targetBaseUrl
|
||||||
|
: ''
|
||||||
|
const conflictMode = job.payload.conflictMode === 'skip' ? 'skip' : 'overwrite'
|
||||||
|
const direction = job.payload.direction === 'pull' ? 'pull' : 'push'
|
||||||
|
if (!targetBaseUrl) throw new Error('SITE_SYNC_JOB_NOT_RESUMABLE')
|
||||||
|
|
||||||
|
registerSiteSyncToken(job.id, token)
|
||||||
|
scheduleBackupJob(job, () => runSiteSyncJob(job.id, {
|
||||||
|
targetBaseUrl,
|
||||||
|
conflictMode,
|
||||||
|
direction
|
||||||
|
}))
|
||||||
|
return getBackupJob(job.id)!
|
||||||
|
}
|
||||||
@@ -1,11 +1,50 @@
|
|||||||
import { getDb } from '../db'
|
import { getDb } from '../db'
|
||||||
|
|
||||||
|
function backupJobsTableSql(): string {
|
||||||
|
const row = getDb().prepare(`
|
||||||
|
SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'backup_jobs'
|
||||||
|
`).get() as { sql?: string } | undefined
|
||||||
|
return row?.sql ?? ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function migrateBackupJobTypes(): void {
|
||||||
|
if (backupJobsTableSql().includes('\'site_sync\'')) return
|
||||||
|
|
||||||
|
const db = getDb()
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE backup_jobs_new (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id INTEGER,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
|
||||||
|
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
|
||||||
|
progress INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total INTEGER NOT NULL DEFAULT 0,
|
||||||
|
message TEXT NOT NULL DEFAULT '',
|
||||||
|
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||||
|
output_path TEXT,
|
||||||
|
error TEXT,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
finished_at TEXT
|
||||||
|
);
|
||||||
|
INSERT INTO backup_jobs_new
|
||||||
|
(id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at)
|
||||||
|
SELECT id, user_id, type, status, progress, total, message, payload_json, output_path, error, created_at, finished_at
|
||||||
|
FROM backup_jobs;
|
||||||
|
DROP TABLE backup_jobs;
|
||||||
|
ALTER TABLE backup_jobs_new RENAME TO backup_jobs;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_backup_jobs_created_at
|
||||||
|
ON backup_jobs(created_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_backup_jobs_user_created
|
||||||
|
ON backup_jobs(user_id, created_at DESC);
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
export function ensureBackupSchema(): void {
|
export function ensureBackupSchema(): void {
|
||||||
getDb().exec(`
|
getDb().exec(`
|
||||||
CREATE TABLE IF NOT EXISTS backup_jobs (
|
CREATE TABLE IF NOT EXISTS backup_jobs (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
user_id INTEGER,
|
user_id INTEGER,
|
||||||
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync')),
|
type TEXT NOT NULL CHECK(type IN ('full_export', 'full_restore', 'backend_sync', 'site_sync')),
|
||||||
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
|
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'done', 'failed')),
|
||||||
progress INTEGER NOT NULL DEFAULT 0,
|
progress INTEGER NOT NULL DEFAULT 0,
|
||||||
total INTEGER NOT NULL DEFAULT 0,
|
total INTEGER NOT NULL DEFAULT 0,
|
||||||
@@ -36,4 +75,5 @@ export function ensureBackupSchema(): void {
|
|||||||
WHERE user_id IS NULL
|
WHERE user_id IS NULL
|
||||||
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
|
AND EXISTS (SELECT 1 FROM users WHERE role = 'admin');
|
||||||
`)
|
`)
|
||||||
|
migrateBackupJobTypes()
|
||||||
}
|
}
|
||||||
@@ -35,6 +35,7 @@ export interface ImageIndexRow {
|
|||||||
backend_id: string
|
backend_id: string
|
||||||
user_id: number | null
|
user_id: number | null
|
||||||
folder: string
|
folder: string
|
||||||
|
media_type: MediaType
|
||||||
original_name: string
|
original_name: string
|
||||||
content_type: string
|
content_type: string
|
||||||
size: number
|
size: number
|
||||||
@@ -71,6 +72,7 @@ function rowToStoredImage(row: ImageIndexRow): StoredImage {
|
|||||||
contentType: row.content_type,
|
contentType: row.content_type,
|
||||||
size: row.size,
|
size: row.size,
|
||||||
userId: row.user_id,
|
userId: row.user_id,
|
||||||
|
mediaType: row.media_type,
|
||||||
hlsKey: row.hls_key ?? null,
|
hlsKey: row.hls_key ?? null,
|
||||||
transcodeStatus: (row.transcode_status ?? 'none') as TranscodeStatus,
|
transcodeStatus: (row.transcode_status ?? 'none') as TranscodeStatus,
|
||||||
transcodeProgress: Number(row.transcode_progress ?? 0),
|
transcodeProgress: Number(row.transcode_progress ?? 0),
|
||||||
@@ -232,7 +234,7 @@ export async function purgeOrphanImageIndexRows(): Promise<number> {
|
|||||||
export function getImageIndexRow(key: string): ImageIndexRow | null {
|
export function getImageIndexRow(key: string): ImageIndexRow | null {
|
||||||
ensureStorageSchema()
|
ensureStorageSchema()
|
||||||
const row = getDb().prepare(`
|
const row = getDb().prepare(`
|
||||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
FROM images
|
FROM images
|
||||||
WHERE key = ?
|
WHERE key = ?
|
||||||
`).get(key) as ImageIndexRow | undefined
|
`).get(key) as ImageIndexRow | undefined
|
||||||
@@ -425,16 +427,30 @@ export function getImageTotals(): { count: number, bytes: number } {
|
|||||||
export function listAllImageIndexRows(): ImageIndexRow[] {
|
export function listAllImageIndexRows(): ImageIndexRow[] {
|
||||||
ensureStorageSchema()
|
ensureStorageSchema()
|
||||||
return getDb().prepare(`
|
return getDb().prepare(`
|
||||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
FROM images
|
FROM images
|
||||||
ORDER BY key ASC
|
ORDER BY key ASC
|
||||||
`).all() as unknown as ImageIndexRow[]
|
`).all() as unknown as ImageIndexRow[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function listImageIndexRowsAfter(
|
||||||
|
afterKey: string,
|
||||||
|
limit: number
|
||||||
|
): ImageIndexRow[] {
|
||||||
|
ensureStorageSchema()
|
||||||
|
return getDb().prepare(`
|
||||||
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
|
FROM images
|
||||||
|
WHERE key > ?
|
||||||
|
ORDER BY key ASC
|
||||||
|
LIMIT ?
|
||||||
|
`).all(afterKey, limit) as unknown as ImageIndexRow[]
|
||||||
|
}
|
||||||
|
|
||||||
export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
|
export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
|
||||||
ensureStorageSchema()
|
ensureStorageSchema()
|
||||||
return getDb().prepare(`
|
return getDb().prepare(`
|
||||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
FROM images
|
FROM images
|
||||||
WHERE user_id = ?
|
WHERE user_id = ?
|
||||||
ORDER BY key ASC
|
ORDER BY key ASC
|
||||||
@@ -444,7 +460,7 @@ export function listImageIndexRowsForUser(userId: number): ImageIndexRow[] {
|
|||||||
export function listImageIndexRowsByBackend(backendId: string): ImageIndexRow[] {
|
export function listImageIndexRowsByBackend(backendId: string): ImageIndexRow[] {
|
||||||
ensureStorageSchema()
|
ensureStorageSchema()
|
||||||
return getDb().prepare(`
|
return getDb().prepare(`
|
||||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
FROM images
|
FROM images
|
||||||
WHERE backend_id = ?
|
WHERE backend_id = ?
|
||||||
ORDER BY key ASC
|
ORDER BY key ASC
|
||||||
@@ -687,7 +703,7 @@ export async function listImages(options: {
|
|||||||
const backendSql = buildBackendFilterSql(options.backendId, params)
|
const backendSql = buildBackendFilterSql(options.backendId, params)
|
||||||
const tagSql = buildTagFilterSql(options.tagFilter, params)
|
const tagSql = buildTagFilterSql(options.tagFilter, params)
|
||||||
const rows = getDb().prepare(`
|
const rows = getDb().prepare(`
|
||||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
FROM images
|
FROM images
|
||||||
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
|
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
|
||||||
ORDER BY uploaded_at DESC, key DESC
|
ORDER BY uploaded_at DESC, key DESC
|
||||||
@@ -733,7 +749,7 @@ export async function searchImages(options: {
|
|||||||
const backendSql = buildBackendFilterSql(options.backendId, params)
|
const backendSql = buildBackendFilterSql(options.backendId, params)
|
||||||
const tagSql = buildTagFilterSql(options.tagFilter, params)
|
const tagSql = buildTagFilterSql(options.tagFilter, params)
|
||||||
const rows = getDb().prepare(`
|
const rows = getDb().prepare(`
|
||||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
FROM images
|
FROM images
|
||||||
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
|
WHERE ${clauses.join(' AND ')}${userSql}${backendSql}${tagSql}
|
||||||
ORDER BY uploaded_at DESC, key DESC
|
ORDER BY uploaded_at DESC, key DESC
|
||||||
@@ -867,7 +883,7 @@ export async function getUserScopedStorageStats(
|
|||||||
export async function listImageKeysForAutoDelete(): Promise<StoredImage[]> {
|
export async function listImageKeysForAutoDelete(): Promise<StoredImage[]> {
|
||||||
ensureStorageSchema()
|
ensureStorageSchema()
|
||||||
const rows = getDb().prepare(`
|
const rows = getDb().prepare(`
|
||||||
SELECT key, backend_id, user_id, folder, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
SELECT key, backend_id, user_id, folder, media_type, original_name, content_type, size, uploaded_at, hls_key, transcode_status, transcode_progress, thumb_key, source_deleted, transcode_preset, transcode_error
|
||||||
FROM images
|
FROM images
|
||||||
WHERE media_type = 'image'
|
WHERE media_type = 'image'
|
||||||
ORDER BY uploaded_at ASC
|
ORDER BY uploaded_at ASC
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ const row: ImageIndexRow = {
|
|||||||
backend_id: 'local',
|
backend_id: 'local',
|
||||||
user_id: 1,
|
user_id: 1,
|
||||||
folder: 'videos',
|
folder: 'videos',
|
||||||
|
media_type: 'video',
|
||||||
original_name: 'example.mp4',
|
original_name: 'example.mp4',
|
||||||
content_type: 'video/mp4',
|
content_type: 'video/mp4',
|
||||||
size: 100,
|
size: 100,
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import { findUserByUsername } from '../db'
|
||||||
|
import {
|
||||||
|
getImageIndexRow,
|
||||||
|
insertImageIndex,
|
||||||
|
updateImageThumb,
|
||||||
|
updateImageTranscode
|
||||||
|
} from '../image-index'
|
||||||
|
import { getActiveBackend } from '../storage/resolver'
|
||||||
|
import {
|
||||||
|
createTag,
|
||||||
|
findTagByName,
|
||||||
|
setImageTags
|
||||||
|
} from '../tags'
|
||||||
|
import type { SiteSyncMediaTransfer } from './protocol'
|
||||||
|
|
||||||
|
export async function applySiteSyncMedia(
|
||||||
|
media: SiteSyncMediaTransfer,
|
||||||
|
conflictMode: 'skip' | 'overwrite',
|
||||||
|
actorUserId: number
|
||||||
|
): Promise<{ success: true, skipped: boolean }> {
|
||||||
|
if (conflictMode === 'skip' && getImageIndexRow(media.key)) {
|
||||||
|
return { success: true, skipped: true }
|
||||||
|
}
|
||||||
|
|
||||||
|
const owner = media.ownerUsername
|
||||||
|
? findUserByUsername(media.ownerUsername)
|
||||||
|
: null
|
||||||
|
if (media.ownerUsername && !owner) {
|
||||||
|
throw new Error(`当前站点缺少用户:${media.ownerUsername}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const backend = await getActiveBackend()
|
||||||
|
const requiredKeys = [
|
||||||
|
...(!media.sourceDeleted ? [media.key] : []),
|
||||||
|
...(media.thumbKey ? [media.thumbKey] : []),
|
||||||
|
...(media.hlsKey ? [media.hlsKey] : [])
|
||||||
|
]
|
||||||
|
for (const key of requiredKeys) {
|
||||||
|
if (!await backend.head(key)) {
|
||||||
|
throw new Error(`同步对象尚未写入:${key}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
insertImageIndex({
|
||||||
|
key: media.key,
|
||||||
|
backendId: backend.id,
|
||||||
|
userId: owner?.id ?? null,
|
||||||
|
originalName: media.originalName,
|
||||||
|
contentType: media.contentType,
|
||||||
|
size: media.size,
|
||||||
|
uploadedAt: media.uploadedAt,
|
||||||
|
mediaType: media.mediaType
|
||||||
|
})
|
||||||
|
|
||||||
|
if (media.thumbKey) updateImageThumb(media.key, media.thumbKey)
|
||||||
|
if (media.mediaType === 'video') {
|
||||||
|
updateImageTranscode(media.key, {
|
||||||
|
status: media.transcodeStatus,
|
||||||
|
hlsKey: media.hlsKey,
|
||||||
|
progress: media.transcodeProgress,
|
||||||
|
preset: media.transcodePreset,
|
||||||
|
sourceDeleted: media.sourceDeleted,
|
||||||
|
error: media.transcodeError
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const tagOwnerId = owner?.id ?? actorUserId
|
||||||
|
const tagIds: number[] = []
|
||||||
|
for (const tag of media.tags) {
|
||||||
|
const existing = findTagByName(tagOwnerId, tag.name)
|
||||||
|
const item = existing
|
||||||
|
? { id: existing.id }
|
||||||
|
: createTag(tagOwnerId, tag.name, tag.color)
|
||||||
|
tagIds.push(item.id)
|
||||||
|
}
|
||||||
|
setImageTags(media.key, tagIds, tagOwnerId, true)
|
||||||
|
|
||||||
|
return { success: true, skipped: false }
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
import type { ImageIndexRow } from '../image-index'
|
||||||
|
import { getTagsForImageKeys } from '../tags'
|
||||||
|
import type {
|
||||||
|
SiteSyncMediaTransfer,
|
||||||
|
SiteSyncTag
|
||||||
|
} from './protocol'
|
||||||
|
|
||||||
|
export function toSiteSyncMediaTransfer(
|
||||||
|
row: ImageIndexRow,
|
||||||
|
ownerUsername: string | null,
|
||||||
|
tags: SiteSyncTag[]
|
||||||
|
): SiteSyncMediaTransfer {
|
||||||
|
return {
|
||||||
|
key: row.key,
|
||||||
|
ownerUsername,
|
||||||
|
originalName: row.original_name,
|
||||||
|
contentType: row.content_type,
|
||||||
|
size: row.size,
|
||||||
|
uploadedAt: row.uploaded_at,
|
||||||
|
mediaType: row.media_type,
|
||||||
|
thumbKey: row.thumb_key,
|
||||||
|
hlsKey: row.hls_key,
|
||||||
|
transcodeStatus: row.transcode_status as SiteSyncMediaTransfer['transcodeStatus'],
|
||||||
|
transcodeProgress: Number(row.transcode_progress ?? 0),
|
||||||
|
transcodePreset: row.transcode_preset,
|
||||||
|
transcodeError: row.transcode_error,
|
||||||
|
sourceDeleted: Number(row.source_deleted ?? 0) === 1,
|
||||||
|
tags
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function toSiteSyncMediaTransferWithTags(
|
||||||
|
row: ImageIndexRow,
|
||||||
|
ownerUsername: string | null
|
||||||
|
): SiteSyncMediaTransfer {
|
||||||
|
const tags = getTagsForImageKeys([row.key]).get(row.key) ?? []
|
||||||
|
return toSiteSyncMediaTransfer(row, ownerUsername, tags)
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import type { MediaType, TranscodeStatus } from '../storage/types'
|
||||||
|
|
||||||
|
export interface SiteSyncTag {
|
||||||
|
name: string
|
||||||
|
color: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSyncMediaTransfer {
|
||||||
|
key: string
|
||||||
|
ownerUsername: string | null
|
||||||
|
originalName: string
|
||||||
|
contentType: string
|
||||||
|
size: number
|
||||||
|
uploadedAt: string
|
||||||
|
mediaType: MediaType
|
||||||
|
thumbKey: string | null
|
||||||
|
hlsKey: string | null
|
||||||
|
transcodeStatus: TranscodeStatus
|
||||||
|
transcodeProgress: number
|
||||||
|
transcodePreset: string | null
|
||||||
|
transcodeError: string | null
|
||||||
|
sourceDeleted: boolean
|
||||||
|
tags: SiteSyncTag[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSyncInventory {
|
||||||
|
users: string[]
|
||||||
|
mediaKeys: string[]
|
||||||
|
objects: Array<{ key: string, size: number }>
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSyncMediaAsset {
|
||||||
|
key: string
|
||||||
|
size: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSyncMediaBundle {
|
||||||
|
media: SiteSyncMediaTransfer
|
||||||
|
assets: SiteSyncMediaAsset[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSyncMediaPage {
|
||||||
|
items: SiteSyncMediaBundle[]
|
||||||
|
totalMedia: number
|
||||||
|
nextCursor: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSyncUsers {
|
||||||
|
users: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SiteSyncDirection = 'push' | 'pull'
|
||||||
|
|
||||||
|
export interface SiteSyncObjectOptions {
|
||||||
|
targetBaseUrl: string
|
||||||
|
token: string
|
||||||
|
conflictMode: 'skip' | 'overwrite'
|
||||||
|
direction?: SiteSyncDirection
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
import { fetchSiteSyncJson, retrySiteSyncOperation } from './runner'
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals()
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('fetchSiteSyncJson', () => {
|
||||||
|
it('retries a broken connection up to three attempts', async () => {
|
||||||
|
const fetchMock = vi.fn()
|
||||||
|
.mockRejectedValueOnce(new Error('connection reset'))
|
||||||
|
.mockRejectedValueOnce(new Error('connection reset'))
|
||||||
|
.mockResolvedValueOnce(new Response('{"ok":true}', {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
}))
|
||||||
|
vi.stubGlobal('fetch', fetchMock)
|
||||||
|
|
||||||
|
await expect(fetchSiteSyncJson('http://target/api', 'token')).resolves.toEqual({ ok: true })
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('retries client errors because every sync failure gets three attempts', async () => {
|
||||||
|
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
|
||||||
|
new Response('{"message":"invalid"}', {
|
||||||
|
status: 400,
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
})
|
||||||
|
))
|
||||||
|
vi.stubGlobal('fetch', fetchMock)
|
||||||
|
|
||||||
|
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('invalid')
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('fails after three server errors', async () => {
|
||||||
|
const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(
|
||||||
|
new Response('{"message":"unavailable"}', {
|
||||||
|
status: 503,
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
})
|
||||||
|
))
|
||||||
|
vi.stubGlobal('fetch', fetchMock)
|
||||||
|
|
||||||
|
await expect(fetchSiteSyncJson('http://target/api', 'token')).rejects.toThrow('unavailable')
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('retries local sync failures and stops after the third attempt', async () => {
|
||||||
|
const operation = vi.fn().mockRejectedValue(new Error('database is busy'))
|
||||||
|
|
||||||
|
await expect(retrySiteSyncOperation(operation)).rejects.toThrow('database is busy')
|
||||||
|
expect(operation).toHaveBeenCalledTimes(3)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,641 @@
|
|||||||
|
import { basename } from 'node:path'
|
||||||
|
import { contentTypeFromKey } from '../content-type'
|
||||||
|
import {
|
||||||
|
getImageIndexRow,
|
||||||
|
listAllImageIndexRows,
|
||||||
|
type ImageIndexRow
|
||||||
|
} from '../image-index'
|
||||||
|
import {
|
||||||
|
findUserByUsername,
|
||||||
|
getPrimaryAdminUserId,
|
||||||
|
listUserIdUsernameMap
|
||||||
|
} from '../db'
|
||||||
|
import { collectMediaAssetKeys } from '../media-assets'
|
||||||
|
import {
|
||||||
|
getActiveBackend,
|
||||||
|
getBackendForKey
|
||||||
|
} from '../storage/resolver'
|
||||||
|
import type { StorageBackend } from '../storage/types'
|
||||||
|
import {
|
||||||
|
getBackupJob,
|
||||||
|
markJobDone,
|
||||||
|
markJobFailed,
|
||||||
|
updateBackupJob
|
||||||
|
} from '../backup/jobs'
|
||||||
|
import { applySiteSyncMedia } from './apply'
|
||||||
|
import { toSiteSyncMediaTransferWithTags } from './manifest'
|
||||||
|
import type {
|
||||||
|
SiteSyncDirection,
|
||||||
|
SiteSyncInventory,
|
||||||
|
SiteSyncMediaBundle,
|
||||||
|
SiteSyncMediaPage,
|
||||||
|
SiteSyncUsers
|
||||||
|
} from './protocol'
|
||||||
|
|
||||||
|
export interface SiteSyncOptions {
|
||||||
|
targetBaseUrl: string
|
||||||
|
conflictMode: 'skip' | 'overwrite'
|
||||||
|
direction: SiteSyncDirection
|
||||||
|
}
|
||||||
|
|
||||||
|
const tokens = (globalThis as typeof globalThis & {
|
||||||
|
__pichostSiteSyncTokens?: Map<string, string>
|
||||||
|
}).__pichostSiteSyncTokens ?? new Map<string, string>()
|
||||||
|
;(globalThis as typeof globalThis & {
|
||||||
|
__pichostSiteSyncTokens?: Map<string, string>
|
||||||
|
}).__pichostSiteSyncTokens = tokens
|
||||||
|
|
||||||
|
export function registerSiteSyncToken(jobId: string, token: string): void {
|
||||||
|
tokens.set(jobId, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
||||||
|
const chunks: Buffer[] = []
|
||||||
|
for await (const chunk of stream) {
|
||||||
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
|
||||||
|
}
|
||||||
|
return Buffer.concat(chunks)
|
||||||
|
}
|
||||||
|
|
||||||
|
function targetUrl(baseUrl: string, path: string): string {
|
||||||
|
return `${baseUrl.replace(/\/+$/, '')}${path}`
|
||||||
|
}
|
||||||
|
|
||||||
|
async function responseError(response: Response): Promise<Error> {
|
||||||
|
const payload = await response.json().catch(() => null) as {
|
||||||
|
message?: string
|
||||||
|
statusMessage?: string
|
||||||
|
} | null
|
||||||
|
return new Error(payload?.message || payload?.statusMessage || `目标站点返回 ${response.status}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const SITE_SYNC_MAX_ATTEMPTS = 3
|
||||||
|
|
||||||
|
export async function retrySiteSyncOperation<T>(
|
||||||
|
operation: () => Promise<T>
|
||||||
|
): Promise<T> {
|
||||||
|
let lastError: Error | null = null
|
||||||
|
for (let attempt = 1; attempt <= SITE_SYNC_MAX_ATTEMPTS; attempt++) {
|
||||||
|
try {
|
||||||
|
return await operation()
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error instanceof Error ? error : new Error('两站点同步失败')
|
||||||
|
if (attempt < SITE_SYNC_MAX_ATTEMPTS) {
|
||||||
|
await new Promise(resolve => setTimeout(resolve, attempt * 500))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastError ?? new Error('两站点同步失败')
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchSiteSyncJsonOnce<T>(
|
||||||
|
url: string,
|
||||||
|
token: string,
|
||||||
|
init?: RequestInit
|
||||||
|
): Promise<T> {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
...init,
|
||||||
|
headers: {
|
||||||
|
'Auth-Token': token,
|
||||||
|
...(init?.headers ?? {})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if (!response.ok) throw await responseError(response)
|
||||||
|
return await response.json() as T
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchSiteSyncJson<T>(
|
||||||
|
url: string,
|
||||||
|
token: string,
|
||||||
|
init?: RequestInit
|
||||||
|
): Promise<T> {
|
||||||
|
return retrySiteSyncOperation(() => (
|
||||||
|
fetchSiteSyncJsonOnce<T>(url, token, init)
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadObject(
|
||||||
|
options: SiteSyncOptions,
|
||||||
|
token: string,
|
||||||
|
backend: StorageBackend,
|
||||||
|
row: ImageIndexRow,
|
||||||
|
ownerUsername: string | null,
|
||||||
|
key: string
|
||||||
|
): Promise<void> {
|
||||||
|
const stored = await backend.head(key)
|
||||||
|
if (!stored) throw new Error(`源文件不存在:${key}`)
|
||||||
|
const bytes = await streamToBuffer(await backend.createStream(key))
|
||||||
|
const form = new FormData()
|
||||||
|
const isPrimary = key === row.key
|
||||||
|
form.append('key', key)
|
||||||
|
form.append('conflictMode', options.conflictMode)
|
||||||
|
form.append('ownerUsername', ownerUsername ?? '')
|
||||||
|
form.append('originalName', isPrimary ? row.original_name : basename(key))
|
||||||
|
form.append('uploadedAt', row.uploaded_at)
|
||||||
|
const contentType = isPrimary ? row.content_type : contentTypeFromKey(key)
|
||||||
|
form.append('contentType', contentType)
|
||||||
|
form.append(
|
||||||
|
'file',
|
||||||
|
new Blob([new Uint8Array(bytes)], { type: contentType }),
|
||||||
|
basename(key)
|
||||||
|
)
|
||||||
|
|
||||||
|
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
||||||
|
targetUrl(options.targetBaseUrl, '/api/site-sync/object'),
|
||||||
|
token,
|
||||||
|
{ method: 'POST', body: form }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function downloadSiteSyncObject(
|
||||||
|
options: SiteSyncOptions,
|
||||||
|
token: string,
|
||||||
|
key: string
|
||||||
|
): Promise<Buffer> {
|
||||||
|
const response = await fetch(
|
||||||
|
targetUrl(
|
||||||
|
options.targetBaseUrl,
|
||||||
|
`/api/site-sync/object?key=${encodeURIComponent(key)}`
|
||||||
|
),
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
'Auth-Token': token
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if (!response.ok) throw await responseError(response)
|
||||||
|
return Buffer.from(await response.arrayBuffer())
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSiteSyncPull(
|
||||||
|
jobId: string,
|
||||||
|
options: SiteSyncOptions,
|
||||||
|
token: string
|
||||||
|
): Promise<void> {
|
||||||
|
const job = getBackupJob(jobId)
|
||||||
|
const checkpoint = job?.payload ?? {}
|
||||||
|
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
|
||||||
|
? checkpoint.lastMediaKey
|
||||||
|
: ''
|
||||||
|
let scanned = Number(checkpoint.scanned ?? 0)
|
||||||
|
let added = Number(checkpoint.added ?? 0)
|
||||||
|
let skipped = Number(checkpoint.skipped ?? 0)
|
||||||
|
let filesSynced = Number(checkpoint.filesSynced ?? 0)
|
||||||
|
let totalFiles = Number(checkpoint.totalFiles ?? 0)
|
||||||
|
let totalMedia = Number(checkpoint.totalMedia ?? job?.total ?? 0)
|
||||||
|
const failedKeys = new Set(
|
||||||
|
Array.isArray(checkpoint.failedKeys)
|
||||||
|
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
|
||||||
|
: []
|
||||||
|
)
|
||||||
|
|
||||||
|
const saveCheckpoint = (message: string) => {
|
||||||
|
updateBackupJob(jobId, {
|
||||||
|
progress: scanned,
|
||||||
|
total: totalMedia,
|
||||||
|
message,
|
||||||
|
payload: {
|
||||||
|
targetBaseUrl: options.targetBaseUrl,
|
||||||
|
conflictMode: options.conflictMode,
|
||||||
|
direction: 'pull',
|
||||||
|
lastMediaKey,
|
||||||
|
scanned,
|
||||||
|
added,
|
||||||
|
skipped,
|
||||||
|
failed: failedKeys.size,
|
||||||
|
filesSynced,
|
||||||
|
totalFiles,
|
||||||
|
totalMedia,
|
||||||
|
failedKeys: [...failedKeys],
|
||||||
|
resumable: true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const progressMessage = (suffix = '') => (
|
||||||
|
`扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}${suffix}`
|
||||||
|
)
|
||||||
|
const failAndCheckpoint = (error: unknown) => {
|
||||||
|
const message = progressMessage(' · 已保存断点')
|
||||||
|
saveCheckpoint(message)
|
||||||
|
markJobFailed(
|
||||||
|
jobId,
|
||||||
|
error instanceof Error ? error.message : '从目标站点拉取失败',
|
||||||
|
message
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const targetUsers = new Set(
|
||||||
|
(await fetchSiteSyncJson<SiteSyncUsers>(
|
||||||
|
targetUrl(options.targetBaseUrl, '/api/site-sync/users'),
|
||||||
|
token
|
||||||
|
)).users
|
||||||
|
)
|
||||||
|
const localUserMap = listUserIdUsernameMap()
|
||||||
|
const localUsers = new Set(localUserMap.values())
|
||||||
|
const missingUsers = [...targetUsers].filter(name => !localUsers.has(name))
|
||||||
|
if (missingUsers.length) {
|
||||||
|
throw new Error(`当前站点缺少同名用户:${missingUsers.join('、')}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const backend = await retrySiteSyncOperation(() => getActiveBackend())
|
||||||
|
const actorUserId = job?.userId ?? getPrimaryAdminUserId() ?? 0
|
||||||
|
const readMediaPage = (path: string) => fetchSiteSyncJson<SiteSyncMediaPage>(
|
||||||
|
targetUrl(options.targetBaseUrl, path),
|
||||||
|
token
|
||||||
|
)
|
||||||
|
|
||||||
|
const processBundleOnce = async (
|
||||||
|
bundle: SiteSyncMediaBundle,
|
||||||
|
isRetry: boolean
|
||||||
|
): Promise<'added' | 'skipped'> => {
|
||||||
|
const { media, assets } = bundle
|
||||||
|
if (
|
||||||
|
!isRetry
|
||||||
|
&& options.conflictMode === 'skip'
|
||||||
|
&& getImageIndexRow(media.key)
|
||||||
|
) {
|
||||||
|
return 'skipped'
|
||||||
|
}
|
||||||
|
|
||||||
|
const owner = media.ownerUsername
|
||||||
|
? findUserByUsername(media.ownerUsername)
|
||||||
|
: null
|
||||||
|
for (const asset of assets) {
|
||||||
|
if (
|
||||||
|
options.conflictMode === 'skip'
|
||||||
|
&& await backend.head(asset.key)
|
||||||
|
) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const bytes = await downloadSiteSyncObject(
|
||||||
|
options,
|
||||||
|
token,
|
||||||
|
asset.key
|
||||||
|
)
|
||||||
|
if (bytes.length !== asset.size) {
|
||||||
|
throw new Error(
|
||||||
|
`同步对象大小不一致:${asset.key},预期 ${asset.size},实际 ${bytes.length}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const isPrimary = asset.key === media.key
|
||||||
|
await backend.put(asset.key, new Uint8Array(bytes), {
|
||||||
|
originalName: isPrimary ? media.originalName : basename(asset.key),
|
||||||
|
uploadedAt: media.uploadedAt,
|
||||||
|
contentType: isPrimary
|
||||||
|
? media.contentType
|
||||||
|
: contentTypeFromKey(asset.key),
|
||||||
|
size: bytes.length,
|
||||||
|
userId: owner?.id ?? null,
|
||||||
|
mediaType: isPrimary
|
||||||
|
? media.mediaType
|
||||||
|
: asset.key.startsWith('videos/')
|
||||||
|
? 'video'
|
||||||
|
: asset.key.startsWith('files/')
|
||||||
|
? 'file'
|
||||||
|
: 'image'
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
await applySiteSyncMedia(media, 'overwrite', actorUserId)
|
||||||
|
return 'added'
|
||||||
|
}
|
||||||
|
const processBundle = (bundle: SiteSyncMediaBundle, isRetry: boolean) => (
|
||||||
|
retrySiteSyncOperation(() => processBundleOnce(bundle, isRetry))
|
||||||
|
)
|
||||||
|
const recordOutcome = (outcome: 'added' | 'skipped', fileCount: number) => {
|
||||||
|
if (outcome === 'added') added += 1
|
||||||
|
else skipped += 1
|
||||||
|
filesSynced += fileCount
|
||||||
|
totalFiles = Math.max(totalFiles, filesSynced)
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const key of [...failedKeys]) {
|
||||||
|
try {
|
||||||
|
const page = await readMediaPage(
|
||||||
|
`/api/site-sync/media?key=${encodeURIComponent(key)}`
|
||||||
|
)
|
||||||
|
const bundle = page.items[0]
|
||||||
|
if (!bundle) throw new Error(`目标媒体不存在:${key}`)
|
||||||
|
const outcome = await processBundle(bundle, true)
|
||||||
|
recordOutcome(outcome, bundle.assets.length)
|
||||||
|
failedKeys.delete(key)
|
||||||
|
if (key > lastMediaKey) lastMediaKey = key
|
||||||
|
saveCheckpoint(progressMessage())
|
||||||
|
} catch (error) {
|
||||||
|
failedKeys.add(key)
|
||||||
|
failAndCheckpoint(error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let cursor = lastMediaKey
|
||||||
|
let lastCheckpointAt = 0
|
||||||
|
while (true) {
|
||||||
|
let page: SiteSyncMediaPage
|
||||||
|
try {
|
||||||
|
page = await readMediaPage(
|
||||||
|
`/api/site-sync/media?limit=50${cursor ? `&after=${encodeURIComponent(cursor)}` : ''}`
|
||||||
|
)
|
||||||
|
} catch (error) {
|
||||||
|
failAndCheckpoint(error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
totalMedia = Math.max(totalMedia, page.totalMedia)
|
||||||
|
|
||||||
|
for (const bundle of page.items) {
|
||||||
|
scanned += 1
|
||||||
|
try {
|
||||||
|
const outcome = await processBundle(bundle, false)
|
||||||
|
recordOutcome(outcome, bundle.assets.length)
|
||||||
|
failedKeys.delete(bundle.media.key)
|
||||||
|
} catch (error) {
|
||||||
|
failedKeys.add(bundle.media.key)
|
||||||
|
lastMediaKey = bundle.media.key
|
||||||
|
failAndCheckpoint(error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
lastMediaKey = bundle.media.key
|
||||||
|
const now = Date.now()
|
||||||
|
if (
|
||||||
|
now - lastCheckpointAt >= 1000
|
||||||
|
|| scanned === totalMedia
|
||||||
|
|| !page.nextCursor
|
||||||
|
) {
|
||||||
|
saveCheckpoint(progressMessage())
|
||||||
|
lastCheckpointAt = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!page.nextCursor) break
|
||||||
|
cursor = page.nextCursor
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failedKeys.size) {
|
||||||
|
markJobFailed(
|
||||||
|
jobId,
|
||||||
|
`${failedKeys.size} 条媒体拉取失败`,
|
||||||
|
progressMessage()
|
||||||
|
)
|
||||||
|
saveCheckpoint(progressMessage())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
totalFiles = filesSynced
|
||||||
|
markJobDone(jobId, {
|
||||||
|
message: `扫描 ${scanned}/${totalMedia || scanned} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
|
||||||
|
payload: {
|
||||||
|
targetBaseUrl: options.targetBaseUrl,
|
||||||
|
conflictMode: options.conflictMode,
|
||||||
|
direction: 'pull',
|
||||||
|
lastMediaKey,
|
||||||
|
scanned,
|
||||||
|
added,
|
||||||
|
skipped,
|
||||||
|
failed: 0,
|
||||||
|
filesSynced,
|
||||||
|
totalFiles,
|
||||||
|
totalMedia: totalMedia || scanned,
|
||||||
|
failedKeys: [],
|
||||||
|
resumable: false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
failAndCheckpoint(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runSiteSyncJob(
|
||||||
|
jobId: string,
|
||||||
|
options: SiteSyncOptions
|
||||||
|
): Promise<void> {
|
||||||
|
const token = tokens.get(jobId)
|
||||||
|
if (!token) throw new Error('同步 Token 不可用,请重新发起任务')
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (options.direction === 'pull') {
|
||||||
|
await runSiteSyncPull(jobId, options, token)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const inventory = await fetchSiteSyncJson<SiteSyncInventory>(
|
||||||
|
targetUrl(options.targetBaseUrl, '/api/site-sync/inventory'),
|
||||||
|
token
|
||||||
|
)
|
||||||
|
const targetUsers = new Set(inventory.users)
|
||||||
|
const targetMedia = new Set(inventory.mediaKeys)
|
||||||
|
const targetObjects = new Map(inventory.objects.map(item => [item.key, item.size]))
|
||||||
|
const rows = listAllImageIndexRows()
|
||||||
|
const ownerMap = listUserIdUsernameMap()
|
||||||
|
const job = getBackupJob(jobId)
|
||||||
|
const checkpoint = job?.payload ?? {}
|
||||||
|
let lastMediaKey = typeof checkpoint.lastMediaKey === 'string'
|
||||||
|
? checkpoint.lastMediaKey
|
||||||
|
: ''
|
||||||
|
let scanned = Number(checkpoint.scanned ?? 0)
|
||||||
|
let added = Number(checkpoint.added ?? 0)
|
||||||
|
let skipped = Number(checkpoint.skipped ?? 0)
|
||||||
|
let filesSynced = Number(checkpoint.filesSynced ?? 0)
|
||||||
|
const failedKeys = new Set(
|
||||||
|
Array.isArray(checkpoint.failedKeys)
|
||||||
|
? checkpoint.failedKeys.filter((key): key is string => typeof key === 'string')
|
||||||
|
: []
|
||||||
|
)
|
||||||
|
|
||||||
|
const missingUsers = [...new Set(
|
||||||
|
rows
|
||||||
|
.map(row => row.user_id == null ? null : ownerMap.get(row.user_id))
|
||||||
|
.filter((name): name is string => Boolean(name))
|
||||||
|
)].filter(name => !targetUsers.has(name))
|
||||||
|
if (missingUsers.length) {
|
||||||
|
throw new Error(`目标站点缺少同名用户:${missingUsers.join('、')}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const plans: Array<{
|
||||||
|
row: ImageIndexRow
|
||||||
|
ownerUsername: string | null
|
||||||
|
backend: StorageBackend
|
||||||
|
keys: string[]
|
||||||
|
}> = []
|
||||||
|
let totalFiles = 0
|
||||||
|
updateBackupJob(jobId, {
|
||||||
|
total: rows.length,
|
||||||
|
progress: scanned,
|
||||||
|
message: '正在统计同步文件…'
|
||||||
|
})
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const backend = await getBackendForKey(row.key)
|
||||||
|
let assetKeys: string[]
|
||||||
|
try {
|
||||||
|
assetKeys = await collectMediaAssetKeys(row, backend)
|
||||||
|
} catch {
|
||||||
|
assetKeys = [
|
||||||
|
...(!Number(row.source_deleted ?? 0) ? [row.key] : []),
|
||||||
|
...(row.thumb_key ? [row.thumb_key] : []),
|
||||||
|
...(row.hls_key ? [row.hls_key] : [])
|
||||||
|
]
|
||||||
|
}
|
||||||
|
const keys = [...new Set(assetKeys)]
|
||||||
|
totalFiles += keys.length
|
||||||
|
plans.push({
|
||||||
|
row,
|
||||||
|
ownerUsername: row.user_id == null ? null : ownerMap.get(row.user_id) ?? null,
|
||||||
|
backend,
|
||||||
|
keys
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const saveCheckpoint = (message: string) => {
|
||||||
|
updateBackupJob(jobId, {
|
||||||
|
progress: scanned,
|
||||||
|
total: rows.length,
|
||||||
|
message,
|
||||||
|
payload: {
|
||||||
|
targetBaseUrl: options.targetBaseUrl,
|
||||||
|
conflictMode: options.conflictMode,
|
||||||
|
direction: 'push',
|
||||||
|
lastMediaKey,
|
||||||
|
scanned,
|
||||||
|
added,
|
||||||
|
skipped,
|
||||||
|
failed: failedKeys.size,
|
||||||
|
filesSynced,
|
||||||
|
totalFiles,
|
||||||
|
failedKeys: [...failedKeys],
|
||||||
|
resumable: true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
plans.sort((a, b) => (
|
||||||
|
a.row.key < b.row.key
|
||||||
|
? -1
|
||||||
|
: a.row.key > b.row.key
|
||||||
|
? 1
|
||||||
|
: 0
|
||||||
|
))
|
||||||
|
lastMediaKey = lastMediaKey || ''
|
||||||
|
let lastCheckpointAt = 0
|
||||||
|
const processPlanOnce = async (plan: typeof plans[number], isRetry: boolean) => {
|
||||||
|
const { row, ownerUsername, backend, keys } = plan
|
||||||
|
if (
|
||||||
|
!isRetry
|
||||||
|
&& options.conflictMode === 'skip'
|
||||||
|
&& targetMedia.has(row.key)
|
||||||
|
) {
|
||||||
|
skipped += 1
|
||||||
|
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
|
||||||
|
failedKeys.delete(row.key)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const key of keys) {
|
||||||
|
const stored = await backend.head(key)
|
||||||
|
if (!stored) throw new Error(`源文件不存在:${key}`)
|
||||||
|
if (
|
||||||
|
options.conflictMode === 'skip'
|
||||||
|
&& targetObjects.get(key) === stored.size
|
||||||
|
) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
await uploadObject(options, token, backend, row, ownerUsername, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
await fetchSiteSyncJsonOnce<{ success: boolean }>(
|
||||||
|
targetUrl(options.targetBaseUrl, '/api/site-sync/media'),
|
||||||
|
token,
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
media: toSiteSyncMediaTransferWithTags(row, ownerUsername),
|
||||||
|
conflictMode: options.conflictMode
|
||||||
|
})
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
failedKeys.delete(row.key)
|
||||||
|
added += 1
|
||||||
|
filesSynced = Math.min(totalFiles, filesSynced + keys.length)
|
||||||
|
}
|
||||||
|
const processPlan = (
|
||||||
|
plan: typeof plans[number],
|
||||||
|
isRetry: boolean
|
||||||
|
) => retrySiteSyncOperation(() => processPlanOnce(plan, isRetry))
|
||||||
|
|
||||||
|
const retryPlans = plans.filter(plan => failedKeys.has(plan.row.key))
|
||||||
|
for (const plan of retryPlans) {
|
||||||
|
try {
|
||||||
|
await processPlan(plan, true)
|
||||||
|
lastMediaKey = plan.row.key
|
||||||
|
} catch {
|
||||||
|
failedKeys.add(plan.row.key)
|
||||||
|
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
|
||||||
|
saveCheckpoint(message)
|
||||||
|
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const plan of plans) {
|
||||||
|
if (plan.row.key <= lastMediaKey) continue
|
||||||
|
scanned += 1
|
||||||
|
try {
|
||||||
|
await processPlan(plan, false)
|
||||||
|
} catch {
|
||||||
|
failedKeys.add(plan.row.key)
|
||||||
|
lastMediaKey = plan.row.key
|
||||||
|
const message = `扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size} · 已保存断点`
|
||||||
|
saveCheckpoint(message)
|
||||||
|
markJobFailed(jobId, '连续 3 次重试失败,已保存断点', message)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
lastMediaKey = plan.row.key
|
||||||
|
const now = Date.now()
|
||||||
|
if (now - lastCheckpointAt >= 1000 || scanned === rows.length || failedKeys.size) {
|
||||||
|
saveCheckpoint(
|
||||||
|
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
|
||||||
|
)
|
||||||
|
lastCheckpointAt = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failedKeys.size) {
|
||||||
|
markJobFailed(
|
||||||
|
jobId,
|
||||||
|
`${failedKeys.size} 条媒体同步失败`,
|
||||||
|
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
|
||||||
|
)
|
||||||
|
saveCheckpoint(
|
||||||
|
`扫描 ${scanned}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 ${failedKeys.size}`
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
markJobDone(jobId, {
|
||||||
|
message: `扫描 ${rows.length}/${rows.length} · 新增 ${added} · 跳过 ${skipped} · 失败 0`,
|
||||||
|
payload: {
|
||||||
|
targetBaseUrl: options.targetBaseUrl,
|
||||||
|
conflictMode: options.conflictMode,
|
||||||
|
direction: 'push',
|
||||||
|
lastMediaKey,
|
||||||
|
scanned: rows.length,
|
||||||
|
added,
|
||||||
|
skipped,
|
||||||
|
failed: 0,
|
||||||
|
filesSynced: totalFiles,
|
||||||
|
totalFiles,
|
||||||
|
failedKeys: [],
|
||||||
|
resumable: false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : '两站点同步失败'
|
||||||
|
markJobFailed(jobId, message)
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
tokens.delete(jobId)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user