Slave/metadata pipeline #6

Merged
nobswebdev merged 9 commits from slave/metadata-pipeline into master 2026-10-18 16:57:39 +00:00
11 changed files with 305 additions and 32 deletions
Showing only changes of commit 1826cef8fe - Show all commits
@@ -5,7 +5,6 @@
* even if exit code is 0 (no stdout / no strip bytes returned). * even if exit code is 0 (no stdout / no strip bytes returned).
* *
* Fix: remove the two stderr early-return checks in the published bundle. * Fix: remove the two stderr early-return checks in the published bundle.
* Upload safety still relies on post-strip re-read + isCleanAfterStrip (app code).
* *
* Upstream: https://github.com/6over3/exiftool (issue: warnings incorrectly treated as errors) * Upstream: https://github.com/6over3/exiftool (issue: warnings incorrectly treated as errors)
* *
@@ -5,18 +5,55 @@
<p v-if="fileName" class="mt-0.5 truncate text-xs text-muted-foreground">{{ fileName }}</p> <p v-if="fileName" class="mt-0.5 truncate text-xs text-muted-foreground">{{ fileName }}</p>
</header> </header>
<div class="flex flex-1 flex-col p-4"> <div class="flex flex-1 flex-col gap-3 p-4">
<p v-if="!fileName" class="text-sm text-muted-foreground">Select an image to view its metadata.</p> <p v-if="!fileName" class="text-sm text-muted-foreground">Select an image to view its metadata.</p>
<p v-else-if="entry.status === 'error'" class="text-sm text-destructive"> <p v-else-if="entry.status === 'error'" class="text-sm text-destructive">
{{ entry.message }} {{ entry.message }}
</p> </p>
<pre <template v-else-if="entry.status === 'ready'">
v-else-if="entry.status === 'ready'" <div
class="max-h-96 flex-1 overflow-auto rounded-md bg-muted p-3 text-xs leading-relaxed whitespace-pre-wrap text-foreground" class="inline-flex w-fit rounded-md border border-border p-0.5"
>{{ entry.output }}</pre role="tablist"
> aria-label="Metadata view"
>
<button
type="button"
role="tab"
class="rounded px-3 py-1.5 text-xs font-medium transition-colors"
:class="
activeTab === 'original'
? 'bg-muted text-foreground'
: 'text-muted-foreground hover:text-foreground'
"
:aria-selected="activeTab === 'original'"
@click="activeTab = 'original'"
>
Original
</button>
<button
type="button"
role="tab"
class="rounded px-3 py-1.5 text-xs font-medium transition-colors"
:class="
activeTab === 'stripped'
? 'bg-muted text-foreground'
: 'text-muted-foreground hover:text-foreground'
"
:aria-selected="activeTab === 'stripped'"
@click="activeTab = 'stripped'"
>
Stripped
</button>
</div>
<pre
role="tabpanel"
class="max-h-96 min-h-32 flex-1 overflow-auto rounded-md bg-muted p-3 text-xs leading-relaxed whitespace-pre-wrap text-foreground"
>{{ activeTab === 'original' ? entry.output : entry.afterStripOutput }}</pre
>
</template>
<p v-else class="text-sm text-muted-foreground">{{ entry.message }}</p> <p v-else class="text-sm text-muted-foreground">{{ entry.message }}</p>
</div> </div>
@@ -24,17 +61,27 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import type { PropType } from 'vue'; import { ref, watch, type PropType } from 'vue';
import type { MetadataPreviewEntry } from '@/types/metadata/MetadataPreview'; import type { MetadataPreviewEntry } from '@/types/metadata/MetadataPreview';
import type { MetadataTab } from '@/types/metadata/MetadataTab';
defineProps({ const props = defineProps({
fileName: { fileName: {
type: String as PropType<string | null>, type: String as PropType<string | null>,
required: true default: null
}, },
entry: { entry: {
type: Object as PropType<MetadataPreviewEntry>, type: Object as PropType<MetadataPreviewEntry>,
required: true required: true
} }
}); });
const activeTab = ref<MetadataTab>('original');
watch(
() => props.fileName,
() => {
activeTab.value = 'original';
}
);
</script> </script>
+63 -1
View File
@@ -18,6 +18,11 @@ export const useExiftoolStore = defineStore('exiftool', () => {
{ resolve: (output: string) => void; reject: (error: Error) => void } { resolve: (output: string) => void; reject: (error: Error) => void }
>(); >();
const stripMetadataWaiters = new Map<
string,
{ resolve: (data: ArrayBuffer) => void; reject: (error: Error) => void }
>();
const initializeExiftool = async (): Promise<void> => { const initializeExiftool = async (): Promise<void> => {
if (initStatus.value === 'ready') { if (initStatus.value === 'ready') {
return; return;
@@ -41,6 +46,14 @@ export const useExiftoolStore = defineStore('exiftool', () => {
}); });
worker.value.addEventListener('message', handleWorkerMessage); worker.value.addEventListener('message', handleWorkerMessage);
worker.value.addEventListener('error', event => {
console.error('[exiftool] worker error', event.message, event.error ?? event);
});
worker.value.addEventListener('messageerror', event => {
console.error('[exiftool] worker messageerror', event);
});
} }
worker.value.postMessage({ type: 'init' }); worker.value.postMessage({ type: 'init' });
@@ -66,6 +79,7 @@ export const useExiftoolStore = defineStore('exiftool', () => {
} }
if (message.type === 'init-error') { if (message.type === 'init-error') {
console.error('[exiftool] init-error', message.message);
initStatus.value = 'error'; initStatus.value = 'error';
initErrorMessage.value = message.message; initErrorMessage.value = message.message;
initPromiseReject?.(new Error(message.message)); initPromiseReject?.(new Error(message.message));
@@ -96,6 +110,34 @@ export const useExiftoolStore = defineStore('exiftool', () => {
} }
readMetadataWaiters.delete(message.id); readMetadataWaiters.delete(message.id);
console.error('[exiftool] read-metadata-error', { id: message.id, message: message.message });
waiter.reject(new Error(message.message));
return;
}
if (message.type === 'strip-metadata-done') {
const waiter = stripMetadataWaiters.get(message.id);
if (!waiter) {
return;
}
stripMetadataWaiters.delete(message.id);
waiter.resolve(message.data);
return;
}
if (message.type === 'strip-metadata-error') {
const waiter = stripMetadataWaiters.get(message.id);
if (!waiter) {
return;
}
stripMetadataWaiters.delete(message.id);
console.error('[exiftool] strip-metadata-error', { id: message.id, message: message.message });
waiter.reject(new Error(message.message)); waiter.reject(new Error(message.message));
} }
}; };
@@ -117,10 +159,30 @@ export const useExiftoolStore = defineStore('exiftool', () => {
}); });
}; };
const stripAllMetadata = async (file: File): Promise<Blob> => {
await initializeExiftool();
const id = crypto.randomUUID();
const workerInstance = worker.value;
if (!workerInstance) {
throw new Error('ExifTool worker is unavailable');
}
const data = await new Promise<ArrayBuffer>((resolve, reject) => {
stripMetadataWaiters.set(id, { resolve, reject });
workerInstance.postMessage({ type: 'strip-metadata', id, file });
});
return new Blob([data], { type: file.type || 'application/octet-stream' });
};
return { return {
initStatus, initStatus,
initErrorMessage, initErrorMessage,
initializeExiftool, initializeExiftool,
readMetadata readMetadata,
stripAllMetadata
}; };
}); });
+82 -14
View File
@@ -7,8 +7,9 @@ const EXIFTOOL_PREPARE_ERROR_MESSAGE = 'Could not prepare ExifTool';
const EXIFTOOL_INITIALIZING_MESSAGE = 'Initializing ExifTool'; const EXIFTOOL_INITIALIZING_MESSAGE = 'Initializing ExifTool';
const METADATA_LOADING_MESSAGE = 'Loading metadata'; const METADATA_LOADING_MESSAGE = 'Loading metadata';
const METADATA_LOAD_ERROR_MESSAGE = 'Could not load image metadata'; const METADATA_LOAD_ERROR_MESSAGE = 'Could not load image metadata';
const METADATA_STRIP_ERROR_MESSAGE = 'Could not strip image metadata';
const METADATA_IDLE_MESSAGE = 'Waiting to load metadata'; const METADATA_IDLE_MESSAGE = 'Waiting to load metadata';
const METADATA_READY_MESSAGE = 'Metadata loaded'; const METADATA_READY_MESSAGE = 'Metadata stripped';
const METADATA_IDLE_PREVIEW: MetadataPreviewEntry = { const METADATA_IDLE_PREVIEW: MetadataPreviewEntry = {
status: 'idle', status: 'idle',
@@ -22,11 +23,12 @@ const METADATA_LOADING_PREVIEW: MetadataPreviewEntry = {
export const useMetadataStore = defineStore('metadata', () => { export const useMetadataStore = defineStore('metadata', () => {
const previewsByKey = ref<Record<string, MetadataPreviewEntry>>({}); const previewsByKey = ref<Record<string, MetadataPreviewEntry>>({});
const strippedBlobByKey = ref<Record<string, Blob>>({});
const exiftoolStore = useExiftoolStore(); const exiftoolStore = useExiftoolStore();
const { initStatus: exiftoolInitStatus } = storeToRefs(exiftoolStore); const { initStatus: exiftoolInitStatus } = storeToRefs(exiftoolStore);
const { initializeExiftool, readMetadata } = exiftoolStore; const { initializeExiftool, readMetadata, stripAllMetadata } = exiftoolStore;
const makeKeyFromFile = (file: File): string => `${file.name}\0${file.size}\0${file.lastModified}`; const makeKeyFromFile = (file: File): string => `${file.name}\0${file.size}\0${file.lastModified}`;
@@ -40,6 +42,16 @@ export const useMetadataStore = defineStore('metadata', () => {
return previewsByKey.value[key] ?? METADATA_IDLE_PREVIEW; return previewsByKey.value[key] ?? METADATA_IDLE_PREVIEW;
}; };
const getStrippedBlob = (file?: File | null): Blob | null => {
if (!file) {
return null;
}
const key = makeKeyFromFile(file);
return strippedBlobByKey.value[key] ?? null;
};
const loadPreview = async (file: File): Promise<MetadataPreviewEntry> => { const loadPreview = async (file: File): Promise<MetadataPreviewEntry> => {
const key = makeKeyFromFile(file); const key = makeKeyFromFile(file);
@@ -72,7 +84,9 @@ export const useMetadataStore = defineStore('metadata', () => {
...previewsByKey.value, ...previewsByKey.value,
[key]: METADATA_LOADING_PREVIEW [key]: METADATA_LOADING_PREVIEW
}; };
} catch { } catch (error) {
console.error('[metadata] ExifTool init failed', { file: file.name, error });
const entry: MetadataPreviewEntry = { const entry: MetadataPreviewEntry = {
status: 'error', status: 'error',
message: EXIFTOOL_PREPARE_ERROR_MESSAGE message: EXIFTOOL_PREPARE_ERROR_MESSAGE
@@ -84,19 +98,13 @@ export const useMetadataStore = defineStore('metadata', () => {
} }
} }
let output: string;
try { try {
const output = await readMetadata(file); output = await readMetadata(file);
} catch (error) {
console.error('[metadata] read failed', { file: file.name, error });
const entry: MetadataPreviewEntry = {
status: 'ready',
message: METADATA_READY_MESSAGE,
output
};
previewsByKey.value = { ...previewsByKey.value, [key]: entry };
return entry;
} catch {
const entry: MetadataPreviewEntry = { const entry: MetadataPreviewEntry = {
status: 'error', status: 'error',
message: METADATA_LOAD_ERROR_MESSAGE message: METADATA_LOAD_ERROR_MESSAGE
@@ -106,6 +114,63 @@ export const useMetadataStore = defineStore('metadata', () => {
return entry; return entry;
} }
let strippedBlob: Blob;
try {
strippedBlob = await stripAllMetadata(file);
} catch (error) {
console.error('[metadata] strip failed', {
file: file.name,
type: file.type,
size: file.size,
error
});
const entry: MetadataPreviewEntry = {
status: 'error',
message: METADATA_STRIP_ERROR_MESSAGE
};
previewsByKey.value = { ...previewsByKey.value, [key]: entry };
return entry;
}
const strippedFile = new File([strippedBlob], file.name, {
type: file.type || strippedBlob.type || 'application/octet-stream',
lastModified: file.lastModified
});
let afterStripOutput: string;
try {
afterStripOutput = await readMetadata(strippedFile);
} catch (error) {
console.error('[metadata] read after strip failed', { file: file.name, error });
const entry: MetadataPreviewEntry = {
status: 'error',
message: METADATA_LOAD_ERROR_MESSAGE
};
previewsByKey.value = { ...previewsByKey.value, [key]: entry };
return entry;
}
strippedBlobByKey.value = { ...strippedBlobByKey.value, [key]: strippedBlob };
const entry: MetadataPreviewEntry = {
status: 'ready',
message: METADATA_READY_MESSAGE,
output,
afterStripOutput
};
previewsByKey.value = { ...previewsByKey.value, [key]: entry };
return entry;
}; };
const loadPreviews = (files: readonly File[]): Promise<MetadataPreviewEntry[]> => const loadPreviews = (files: readonly File[]): Promise<MetadataPreviewEntry[]> =>
@@ -113,12 +178,15 @@ export const useMetadataStore = defineStore('metadata', () => {
const clearAllPreviews = (): void => { const clearAllPreviews = (): void => {
previewsByKey.value = {}; previewsByKey.value = {};
strippedBlobByKey.value = {};
}; };
return { return {
previewsByKey, previewsByKey,
strippedBlobByKey,
makeKeyFromFile, makeKeyFromFile,
getPreview, getPreview,
getStrippedBlob,
loadPreview, loadPreview,
loadPreviews, loadPreviews,
clearAllPreviews clearAllPreviews
@@ -2,5 +2,5 @@ export type MetadataPreviewEntry =
| { status: 'idle'; message: string } | { status: 'idle'; message: string }
| { status: 'pending'; message: string } | { status: 'pending'; message: string }
| { status: 'loading'; message: string } | { status: 'loading'; message: string }
| { status: 'ready'; message: string; output: string } | { status: 'ready'; message: string; output: string; afterStripOutput: string }
| { status: 'error'; message: string }; | { status: 'error'; message: string };
@@ -0,0 +1 @@
export type MetadataTab = 'original' | 'stripped';
@@ -1,3 +1,4 @@
export type ExiftoolWorkerInMessage = export type ExiftoolWorkerInMessage =
| { type: 'init' } | { type: 'init' }
| { type: 'read-metadata'; id: string; file: File }; | { type: 'read-metadata'; id: string; file: File }
| { type: 'strip-metadata'; id: string; file: File };
@@ -1,3 +1,4 @@
export type ExiftoolWorkerJob = export type ExiftoolWorkerJob =
| { type: 'init' } | { type: 'init' }
| { type: 'read-metadata'; id: string; file: File }; | { type: 'read-metadata'; id: string; file: File }
| { type: 'strip-metadata'; id: string; file: File };
@@ -2,4 +2,6 @@ export type ExiftoolWorkerOutMessage =
| { type: 'init-done' } | { type: 'init-done' }
| { type: 'init-error'; message: string } | { type: 'init-error'; message: string }
| { type: 'read-metadata-done'; id: string; output: string } | { type: 'read-metadata-done'; id: string; output: string }
| { type: 'read-metadata-error'; id: string; message: string }; | { type: 'read-metadata-error'; id: string; message: string }
| { type: 'strip-metadata-done'; id: string; data: ArrayBuffer }
| { type: 'strip-metadata-error'; id: string; message: string };
+15
View File
@@ -0,0 +1,15 @@
export function toArrayBuffer(data: unknown): ArrayBuffer | null {
if (data instanceof ArrayBuffer) {
return data;
}
if (data instanceof Uint8Array) {
const { buffer, byteOffset, byteLength } = data;
if (buffer instanceof ArrayBuffer) {
return buffer.slice(byteOffset, byteOffset + byteLength);
}
}
return null;
}
+80 -3
View File
@@ -1,9 +1,10 @@
import zeroperlWasmUrl from '@6over3/zeroperl-ts/zeroperl.wasm?url'; import zeroperlWasmUrl from '@6over3/zeroperl-ts/zeroperl.wasm?url';
import { parseMetadata } from '@uswriting/exiftool'; import { parseMetadata, writeMetadata } from '@uswriting/exiftool';
import { EXIFTOOL_WORKER_INIT_JPEG } from '@/consts/exiftoolWorkerInitJpeg'; import { EXIFTOOL_WORKER_INIT_JPEG } from '@/consts/exiftoolWorkerInitJpeg';
import type { ExiftoolWorkerInMessage } from '@/types/workers/exiftoolWorkerInMessage'; import type { ExiftoolWorkerInMessage } from '@/types/workers/exiftoolWorkerInMessage';
import type { ExiftoolWorkerJob } from '@/types/workers/exiftoolWorkerJob'; import type { ExiftoolWorkerJob } from '@/types/workers/exiftoolWorkerJob';
import { resolveErrorMessage } from '@/utils/resolveErrorMessage'; import { resolveErrorMessage } from '@/utils/resolveErrorMessage';
import { toArrayBuffer } from '@/utils/toArrayBuffer';
/** Custom fetch function for `@uswriting/exiftool` to load `zeroperl.wasm` from the worker. */ /** Custom fetch function for `@uswriting/exiftool` to load `zeroperl.wasm` from the worker. */
const exiftoolParseOptions = { fetch: () => fetch(zeroperlWasmUrl) }; const exiftoolParseOptions = { fetch: () => fetch(zeroperlWasmUrl) };
@@ -107,17 +108,87 @@ const processJob = async (job: ExiftoolWorkerJob): Promise<void> => {
const result = await parseMetadata(job.file, exiftoolParseOptions); const result = await parseMetadata(job.file, exiftoolParseOptions);
if (!result.success) { if (!result.success) {
self.postMessage({ type: 'read-metadata-error', id: job.id, message: result.error }); const message = result.error ?? 'Could not load image metadata';
console.error('[exiftool-worker] read failed', { id: job.id, file: job.file.name, message });
self.postMessage({ type: 'read-metadata-error', id: job.id, message });
return; return;
} }
self.postMessage({ type: 'read-metadata-done', id: job.id, output: result.data }); self.postMessage({ type: 'read-metadata-done', id: job.id, output: result.data });
} catch (error) { } catch (error) {
const message = resolveErrorMessage(error, 'Could not load image metadata');
console.error('[exiftool-worker] read threw', { id: job.id, file: job.file.name, error });
self.postMessage({ self.postMessage({
type: 'read-metadata-error', type: 'read-metadata-error',
id: job.id, id: job.id,
message: resolveErrorMessage(error, 'Could not load image metadata') message
});
}
return;
}
if (job.type === 'strip-metadata') {
try {
await initializeWorker();
const result = await writeMetadata(
job.file,
{},
{ args: ['-all='], ...exiftoolParseOptions }
);
if (!result.success) {
const message = result.error ?? 'Could not strip image metadata';
console.error('[exiftool-worker] strip failed', {
id: job.id,
file: job.file.name,
type: job.file.type,
exitCode: result.exitCode,
message
});
self.postMessage({
type: 'strip-metadata-error',
id: job.id,
message
});
return;
}
const data = toArrayBuffer(result.data);
if (!data) {
const message = 'Strip succeeded but returned no image bytes';
console.error('[exiftool-worker] strip missing data', {
id: job.id,
file: job.file.name,
dataType: result.data === null ? 'null' : typeof result.data
});
self.postMessage({ type: 'strip-metadata-error', id: job.id, message });
return;
}
self.postMessage({ type: 'strip-metadata-done', id: job.id, data }, { transfer: [data] });
} catch (error) {
const message = resolveErrorMessage(error, 'Failed to strip metadata');
console.error('[exiftool-worker] strip failed', { id: job.id, file: job.file.name, error });
self.postMessage({
type: 'strip-metadata-error',
id: job.id,
message
}); });
} }
} }
@@ -134,5 +205,11 @@ self.addEventListener('message', (event: MessageEvent<ExiftoolWorkerInMessage>)
if (message.type === 'read-metadata') { if (message.type === 'read-metadata') {
enqueue({ type: 'read-metadata', id: message.id, file: message.file }); enqueue({ type: 'read-metadata', id: message.id, file: message.file });
return;
}
if (message.type === 'strip-metadata') {
enqueue({ type: 'strip-metadata', id: message.id, file: message.file });
} }
}); });