From a6c6aff8644ac73e1e5425f96e713ec2cd849385 Mon Sep 17 00:00:00 2001 From: Laurin Weger Date: Mon, 8 Jul 2024 22:37:22 +0200 Subject: [PATCH] create a worker for upload and downloading with the api (not working since it's operating in a different context) --- ng-app/src/lib/Login.svelte | 4 +- ng-app/src/lib/Test.svelte | 121 +---------- ng-app/src/store.ts | 72 ++++++- ng-app/src/workers/download-upload-worker.ts | 200 ++++++++++++++++++ .../{worker.js => workers/wallet-worker.js} | 2 +- 5 files changed, 283 insertions(+), 116 deletions(-) create mode 100644 ng-app/src/workers/download-upload-worker.ts rename ng-app/src/{worker.js => workers/wallet-worker.js} (94%) diff --git a/ng-app/src/lib/Login.svelte b/ng-app/src/lib/Login.svelte index 765df6b..949739b 100644 --- a/ng-app/src/lib/Login.svelte +++ b/ng-app/src/lib/Login.svelte @@ -193,7 +193,9 @@ trusted, }); } else { - let worker_import = await import("../worker.js?worker&inline"); + let worker_import = await import( + "../workers/wallet-worker.js?worker&inline" + ); const myWorker = new worker_import.default(); myWorker.onerror = (e) => { console.error(e); diff --git a/ng-app/src/lib/Test.svelte b/ng-app/src/lib/Test.svelte index c15b09c..4212c58 100644 --- a/ng-app/src/lib/Test.svelte +++ b/ng-app/src/lib/Test.svelte @@ -22,6 +22,7 @@ cannot_load_offline, online, get_blob, + upload_file, } from "../store"; import { link } from "svelte-spa-router"; import { onMount, onDestroy, tick } from "svelte"; @@ -530,126 +531,20 @@ let fileinput; - function uploadFile(upload_id, nuri, file, success) { - let chunkSize = 1_048_564; - let fileSize = file.size; - let offset = 0; - let readBlock = null; - upload_progress = { total: fileSize, current: offset }; - - let onLoadHandler = async function (event) { - let result = event.target.result; - - if (event.target.error == null) { - offset += result.byteLength; - upload_progress = { total: fileSize, current: offset }; - - // console.log("chunk", result); - - let res = await ng.upload_chunk( - $active_session.session_id, - upload_id, - result, - nuri - ); - //console.log("chunk upload res", res); - // if (onChunkRead) { - // onChunkRead(result); - // } - } else { - // if (onChunkError) { - // onChunkError(event.target.error); - // } - return; - } - - // If finished: - if (offset >= fileSize) { - //console.log("file uploaded"); - let res = await ng.upload_chunk( - $active_session.session_id, - upload_id, - [], - nuri - ); - //console.log("end upload res", res); - if (success) { - upload_progress = { total: fileSize, current: fileSize }; - success(res); - } else { - upload_progress = { total: fileSize, current: fileSize, error: true }; - } - - // Make progress bar disappear - setTimeout(() => { - upload_progress = null; - }, 2_500); - return; - } - - readBlock(offset, chunkSize, file); - }; - - readBlock = function (offset, length, file) { - let fileReader = new FileReader(); - let blob = file.slice(offset, length + offset); - fileReader.onload = onLoadHandler; - fileReader.readAsArrayBuffer(blob); - }; - - readBlock(offset, chunkSize, file); - return; - } - const onFileSelected = async (e) => { let image = e.target.files[0]; if (!image) return; //console.log(image); - let nuri = { - target: "PrivateStore", - entire_store: false, - access: [], - locator: [], - }; - - let start_request = { - V0: { - command: "FilePut", - nuri, - payload: { - V0: { - RandomAccessFilePut: image.type, - }, - }, - session_id: $active_session.session_id, - }, - }; - - let start_res = await ng.app_request(start_request); - let upload_id = start_res.V0.FileUploading; + await upload_file($active_session.session_id, image, (progress) => { + upload_progress = progress; + }); - uploadFile(upload_id, nuri, image, async (reference) => { - if (reference) { - let request = { - V0: { - command: "FilePut", - nuri, - payload: { - V0: { - AddFile: { - filename: image.name, - object: reference.V0.FileUploaded, - }, - }, - }, - session_id: $active_session.session_id, - }, - }; + // Make progress bar disappear + setTimeout(() => { + upload_progress = null; + }, 2_500); - await ng.app_request(request); - } - }); fileinput.value = ""; }; diff --git a/ng-app/src/store.ts b/ng-app/src/store.ts index fd477dc..647b236 100644 --- a/ng-app/src/store.ts +++ b/ng-app/src/store.ts @@ -448,12 +448,21 @@ export const branch_subs = function(nuri) { //console.log("callback unsub"); already_subscribed.decrease(); } - } } }; +const upload_download_worker_import = await import("./workers/download-upload-worker.ts?worker&inline"); +const upload_download_worker = new upload_download_worker_import.default(); +upload_download_worker.addEventListener("error", (event) => { + console.error("An error occurred in the upload-download worker", { cause: event.error }); +}); +upload_download_worker.addEventListener("messageerror", (event) => { + console.error("A message error occurred in the upload-download worker", { event }); +}); + let blob_cache = {}; + export async function get_blob(ref: { nuri: string | number; reference: { key: any; id: any; }; }) { if (!ref) return false; const cached = blob_cache[ref.nuri]; @@ -504,5 +513,66 @@ export async function get_blob(ref: { nuri: string | number; reference: { key: a blob_cache[ref.nuri] = prom; return prom; } +export async function get_blob_new(ref) { + if (!ref) return false; + const cached = blob_cache[ref.nuri]; + if (cached) { + return cached; + } + + let prom = new Promise(async (resolve, reject) => { + const request_id = Math.random().toString(); + + const handle_message = (event) => { + const { data } = event; + // Only handle events with our reference. + if (data.reference != request_id) { + return; + } + + if (data.error) { + reject({ message: "DownloadError", ref, error: data.error }) + } + if (data.result) { + upload_download_worker.removeEventListener("message", handle_message); + resolve(data.result); + } + }; + + upload_download_worker.addEventListener("message", handle_message); + upload_download_worker.postMessage({ request_id, command: "download", params: { ref } }) + + }); + blob_cache[ref.nuri] = prom; + return prom; +} + +export async function upload_file_new(session_id, file: File, on_progress: (progress: { total: number, current: number }) => void) { + return new Promise((resolve, reject) => { + const request_id = Math.random().toString(); + + const handle_message = (event) => { + const { data } = event; + // Only handle events with our reference. + if (data.reference != request_id) { + return; + } + if (data.progress) { + on_progress(data.progress); + return; + } + if (data.error) { + reject({ message: "UploadError", session_id, error: data.error }) + } + if (data.result) { + upload_download_worker.removeEventListener("message", handle_message); + resolve(true); + } + }; + + upload_download_worker.addEventListener("message", handle_message); + upload_download_worker.postMessage({ request_id, command: "upload", params: { session_id, file } }) + }) +} //export default branch_commits; \ No newline at end of file diff --git a/ng-app/src/workers/download-upload-worker.ts b/ng-app/src/workers/download-upload-worker.ts new file mode 100644 index 0000000..ea7cecf --- /dev/null +++ b/ng-app/src/workers/download-upload-worker.ts @@ -0,0 +1,200 @@ +import * as api from "ng-sdk-js"; +import { default as ng } from "../api"; + +//console.log("loaded worker"); + +// Accepts messages with data: +// - command: `get_blob` | `upload` +// - request_id: A request id defined by the caller that is used on `postMessage` together with the `result` of the command or `progress` +// - params: an object of parameters to pass to the function. +// +// May send progress messages with {request_id, progress}, {request_id, result}, or {request_id, error}. + +onmessage = async (e) => { + const { data } = e; + let result; + switch (data.command) { + case "download": + get_blob(data.request_id, data.params); + break; + case "upload": + prepare_upload(data.request_id, data.params) + break; + } +}; + +postMessage({ loaded: true }); + +async function get_blob(request_id, { session_id, ref }) { + if (!ref) return false; + + try { + let nuri = { + target: "PrivateStore", + entire_store: false, + access: [{ Key: ref.reference.key }], + locator: [], + object: ref.reference.id, + }; + + let file_request = { + V0: { + command: "FileGet", + nuri, + session_id: session_id, + }, + }; + + let final_blob; + let content_type; + console.debug("downloading with ng.app_request_stream", ref); + + let unsub = await ng.app_request_stream(file_request, async (blob) => { + console.log("GOT APP RESPONSE", blob); + if (blob.V0.FileMeta) { + content_type = blob.V0.FileMeta.content_type; + final_blob = new Blob([], { type: content_type }); + } else if (blob.V0.FileBinary) { + if (blob.V0.FileBinary.byteLength > 0) { + final_blob = new Blob([final_blob, blob.V0.FileBinary], { + type: content_type, + }); + } + } else if (blob.V0 == "EndOfStream") { + var blobUrl = URL.createObjectURL(final_blob); + postMessage({ request_id, result: blobUrl }); + + return blobUrl; + } + }); + } catch (e) { + console.error(e); + postMessage({ request_id, error: e }); + } +} + +// Upload + +function upload_file(request_id, { session_id, upload_id, nuri, file, success: on_success }) { + let chunkSize = 1_048_564; + let fileSize = file.size; + let offset = 0; + let readBlock = null; + postMessage({ request_id, progress: { total: fileSize, current: offset } }); + + console.log("in upload_file"); + + let onLoadHandler = async function(event) { + let result = event.target.result; + + console.log("onLoadHandler", result); + + if (event.target.error == null) { + offset += result.byteLength; + postMessage({ request_id, progress: { total: fileSize, current: offset } }); + + // console.log("chunk", result); + + let res = await ng.upload_chunk( + session_id, + upload_id, + result, + nuri + ); + //console.log("chunk upload res", res); + // if (onChunkRead) { + // onChunkRead(result); + // } + } else { + // if (onChunkError) { + // onChunkError(event.target.error); + // } + return; + } + + // If finished: + if (offset >= fileSize) { + //console.log("file uploaded"); + let res = await ng.upload_chunk( + session_id, + upload_id, + [], + nuri + ); + + postMessage({ request_id, progress: { total: fileSize, current: fileSize } }); + on_success(res); + + return; + } + + readBlock(offset, chunkSize, file); + }; + + readBlock = function(offset, length, file) { + let fileReader = new FileReader(); + let blob = file.slice(offset, length + offset); + fileReader.onload = onLoadHandler; + fileReader.readAsArrayBuffer(blob); + }; + + readBlock(offset, chunkSize, file); + return; +} + +const prepare_upload = async (request_id, { session_id, file }) => { + if (!file) return; + //console.log(file); + + let nuri = { + target: "PrivateStore", + entire_store: false, + access: [], + locator: [], + }; + + let start_request = { + V0: { + command: "FilePut", + nuri, + payload: { + V0: { + RandomAccessFilePut: file.type, + }, + }, + session_id: session_id, + }, + }; + + console.log("prepare_upload", file); + + let start_res = await ng.app_request(start_request); + let upload_id = start_res.V0.FileUploading; + + console.log("prepare_upload, start_res", start_res); + + upload_file(request_id, { + session_id, upload_id, nuri, file, success: async (reference) => { + if (reference) { + let request = { + V0: { + command: "FilePut", + nuri, + payload: { + V0: { + AddFile: { + filename: file.name, + object: reference.V0.FileUploaded, + }, + }, + }, + session_id: session_id, + }, + }; + + await ng.app_request(request); + postMessage({ request_id, result: "Upload finished" }); + } + } + }); +}; diff --git a/ng-app/src/worker.js b/ng-app/src/workers/wallet-worker.js similarity index 94% rename from ng-app/src/worker.js rename to ng-app/src/workers/wallet-worker.js index c70b842..8386de5 100644 --- a/ng-app/src/worker.js +++ b/ng-app/src/workers/wallet-worker.js @@ -1,5 +1,5 @@ import * as api from "ng-sdk-js"; -import { default as ng } from "./api"; +import { default as ng } from "../api"; //console.log("loaded worker");