jaxoncreed
2 years ago
parent
21d631c406
commit
fa141fed5d
File diff suppressed because it is too large
Load Diff
@ -1,158 +0,0 @@ |
||||
import type { LeafUri } from "./uriTypes"; |
||||
import type { PresentLeaf } from "./resource/abstract/leaf/PresentLeaf"; |
||||
import type { DataLeaf } from "./resource/abstract/leaf/DataLeaf"; |
||||
import type { ResourceError } from "./resource/error/ResourceError"; |
||||
import type { BinaryLeaf } from "./resource/abstract/leaf/BinaryLeaf"; |
||||
import type { DatasetChanges } from "@ldo/rdf-utils"; |
||||
import type { AbsentLeaf } from "./resource/abstract/leaf/AbsentLeaf"; |
||||
|
||||
export interface WaitingProcess<Args extends unknown[], Return> { |
||||
name: string; |
||||
args: Args; |
||||
perform: (...args: Args) => Promise<Return>; |
||||
awaitingResolutions: ((returnValue: Return) => void)[]; |
||||
awaitingRejections: ((err: unknown) => void)[]; |
||||
} |
||||
|
||||
export interface WaitingProcessOptions<Args extends unknown[], Return> { |
||||
name: string; |
||||
args: Args; |
||||
perform: (...args: Args) => Promise<Return>; |
||||
modifyLastProcess: ( |
||||
lastProcess: WaitingProcess<unknown[], unknown>, |
||||
args: Args, |
||||
) => boolean; |
||||
} |
||||
|
||||
export abstract class LeafRequestBatcher { |
||||
private lastRequestTimestampMap: Record<string, number> = {}; |
||||
private isWaiting: boolean = false; |
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
private processQueue: WaitingProcess<any[], any>[] = []; |
||||
private shouldBatchAllRequests: boolean; |
||||
private batchMilis: number; |
||||
|
||||
private triggerOrWaitProcess() { |
||||
const processName = this.shouldBatchAllRequests |
||||
? "any" |
||||
: this.processQueue[0].name; |
||||
|
||||
// Set last request timestamp if not available
|
||||
if (!this.lastRequestTimestampMap[processName]) { |
||||
this.lastRequestTimestampMap[processName] = Date.UTC(0, 0, 0, 0, 0, 0, 0); |
||||
} |
||||
|
||||
const lastRequestTimestamp = this.lastRequestTimestampMap[processName]; |
||||
const timeSinceLastTrigger = Date.now() - lastRequestTimestamp; |
||||
|
||||
const triggerProcess = async () => { |
||||
this.isWaiting = false; |
||||
this.lastRequestTimestampMap[processName] = Date.now(); |
||||
this.lastRequestTimestampMap["any"] = Date.now(); |
||||
const processToTrigger = this.processQueue.shift(); |
||||
if (processToTrigger) { |
||||
try { |
||||
const returnValue = await processToTrigger.perform( |
||||
processToTrigger.args, |
||||
); |
||||
processToTrigger.awaitingResolutions.forEach((callback) => { |
||||
callback(returnValue); |
||||
}); |
||||
} catch (err) { |
||||
processToTrigger.awaitingRejections.forEach((callback) => { |
||||
callback(err); |
||||
}); |
||||
} |
||||
this.triggerOrWaitProcess(); |
||||
} |
||||
}; |
||||
|
||||
if (timeSinceLastTrigger < this.batchMilis && !this.isWaiting) { |
||||
this.isWaiting = true; |
||||
setTimeout(triggerProcess, this.batchMilis - timeSinceLastTrigger); |
||||
} else { |
||||
triggerProcess(); |
||||
} |
||||
} |
||||
|
||||
protected async queueProcess<ReturnType>( |
||||
options: WaitingProcessOptions<unknown[], ReturnType>, |
||||
): Promise<ReturnType> { |
||||
return new Promise((resolve, reject) => { |
||||
const lastProcessInQueue = |
||||
this.processQueue[this.processQueue.length - 1]; |
||||
if (lastProcessInQueue) { |
||||
const didModifyLast = lastProcessInQueue |
||||
? options.modifyLastProcess(lastProcessInQueue, options.args) |
||||
: false; |
||||
if (didModifyLast) { |
||||
lastProcessInQueue.awaitingResolutions.push(resolve); |
||||
lastProcessInQueue.awaitingRejections.push(reject); |
||||
return; |
||||
} |
||||
} |
||||
this.processQueue.push({ |
||||
name: options.name, |
||||
args: options.args, |
||||
perform: options.perform, |
||||
awaitingResolutions: [resolve], |
||||
awaitingRejections: [reject], |
||||
}); |
||||
this.triggerOrWaitProcess(); |
||||
|
||||
// const READ_KEY = "read";
|
||||
// const lastProcessInQueue =
|
||||
// this.processQueue[this.processQueue.length - 1];
|
||||
// if (lastProcessInQueue?.name === READ_KEY) {
|
||||
// lastProcessInQueue.awaitingResolutions.push(resolve);
|
||||
// lastProcessInQueue.awaitingRejections.push(reject);
|
||||
// } else {
|
||||
// const readProcess: WaitingProcess<[], PresentLeaf | ResourceError> = {
|
||||
// name: READ_KEY,
|
||||
// args: [],
|
||||
// perform: this.performRead,
|
||||
// awaitingResolutions: [resolve],
|
||||
// awaitingRejections: [reject],
|
||||
// };
|
||||
// this.processQueue.push(readProcess);
|
||||
// this.triggerOrWaitProcess();
|
||||
// }
|
||||
}); |
||||
} |
||||
|
||||
// All intance variables
|
||||
uri: LeafUri; |
||||
|
||||
// Read Methods
|
||||
read(): Promise<PresentLeaf | ResourceError> { |
||||
const READ_KEY = "read"; |
||||
return this.queueProcess({ |
||||
name: READ_KEY, |
||||
args: [], |
||||
perform: this.performRead, |
||||
modifyLastProcess: (last) => { |
||||
return last.name === READ_KEY; |
||||
}, |
||||
}); |
||||
} |
||||
|
||||
private performRead(): Promise<PresentLeaf | ResourceError> { |
||||
console.log("Reading"); |
||||
throw new Error("Doing Read"); |
||||
} |
||||
|
||||
// Create Methods
|
||||
abstract createData(overwrite?: boolean): Promise<DataLeaf | ResourceError>; |
||||
|
||||
abstract upload( |
||||
blob: Blob, |
||||
mimeType: string, |
||||
overwrite?: boolean, |
||||
): Promise<BinaryLeaf | ResourceError>; |
||||
|
||||
abstract updateData( |
||||
changes: DatasetChanges, |
||||
): Promise<DataLeaf | ResourceError>; |
||||
|
||||
abstract delete(): Promise<AbsentLeaf | ResourceError>; |
||||
} |
||||
@ -1,9 +0,0 @@ |
||||
interface loadingStatus; |
||||
|
||||
export class LoadingManager { |
||||
private loadingStatus: Record<string, Array<[key: string, promise: Promise<unknown>]>> |
||||
|
||||
public registerProcess(): void { |
||||
|
||||
} |
||||
} |
||||
@ -1,15 +1,15 @@ |
||||
import type TypedEmitter from "typed-emitter"; |
||||
// import type TypedEmitter from "typed-emitter";
|
||||
import type { SolidLdoDataset } from "./SolidLdoDataset"; |
||||
import type { DocumentError } from "./document/errors/DocumentError"; |
||||
// import type { DocumentError } from "./document/errors/DocumentError";
|
||||
|
||||
export type OnDocumentErrorCallback = (error: DocumentError) => void; |
||||
// export type OnDocumentErrorCallback = (error: DocumentError) => void;
|
||||
|
||||
export type DocumentEventEmitter = TypedEmitter<{ |
||||
documentError: OnDocumentErrorCallback; |
||||
}>; |
||||
// export type DocumentEventEmitter = TypedEmitter<{
|
||||
// documentError: OnDocumentErrorCallback;
|
||||
// }>;
|
||||
|
||||
export interface SolidLdoDatasetContext { |
||||
solidLdoDataset: SolidLdoDataset; |
||||
documentEventEmitter: DocumentEventEmitter; |
||||
// documentEventEmitter: DocumentEventEmitter;
|
||||
fetch: typeof fetch; |
||||
} |
||||
|
||||
@ -0,0 +1,125 @@ |
||||
import type { LeafUri } from "../uriTypes"; |
||||
import { RequestBatcher } from "../util/RequestBatcher"; |
||||
import type { SolidLdoDatasetContext } from "../SolidLdoDatasetContext"; |
||||
import { AbsentResult } from "./requesterResults/AbsentResult"; |
||||
import { |
||||
DataResult, |
||||
TurtleFormattingError, |
||||
} from "./requesterResults/DataResult"; |
||||
import { BinaryResult } from "./requesterResults/BinaryResult"; |
||||
import { |
||||
HttpErrorResult, |
||||
ServerHttpError, |
||||
UnauthenticatedHttpError, |
||||
UnexpectedHttpError, |
||||
} from "./requesterResults/HttpErrorResult"; |
||||
import { UnexpectedError } from "./requesterResults/ErrorResult"; |
||||
import { parseRdf } from "@ldo/ldo"; |
||||
import { namedNode } from "@rdfjs/data-model"; |
||||
|
||||
export type ReadResult = |
||||
| AbsentResult |
||||
| DataResult |
||||
| BinaryResult |
||||
| ServerHttpError |
||||
| UnauthenticatedHttpError |
||||
| UnexpectedHttpError |
||||
| UnexpectedError |
||||
| TurtleFormattingError; |
||||
|
||||
export class LeafRequester { |
||||
private requestBatcher = new RequestBatcher(); |
||||
|
||||
// All intance variables
|
||||
readonly uri: LeafUri; |
||||
private context: SolidLdoDatasetContext; |
||||
|
||||
constructor(uri: LeafUri, context: SolidLdoDatasetContext) { |
||||
this.uri = uri; |
||||
this.context = context; |
||||
} |
||||
|
||||
// Read Methods
|
||||
read(): Promise<ReadResult> { |
||||
const READ_KEY = "read"; |
||||
return this.requestBatcher.queueProcess({ |
||||
name: READ_KEY, |
||||
args: [], |
||||
perform: this.performRead.bind(this), |
||||
modifyQueue: (queue, isLoading) => { |
||||
if (queue.length === 0) { |
||||
return isLoading[READ_KEY]; |
||||
} else { |
||||
return queue[queue.length - 1].name === READ_KEY; |
||||
} |
||||
}, |
||||
}); |
||||
} |
||||
|
||||
private async performRead(): Promise<ReadResult> { |
||||
try { |
||||
// Fetch options to determine the document type
|
||||
const response = await this.context.fetch(this.uri); |
||||
if (AbsentResult.is(response)) { |
||||
return new AbsentResult(this.uri); |
||||
} |
||||
if (ServerHttpError.is(response)) { |
||||
return new ServerHttpError(this.uri, response); |
||||
} |
||||
if (UnauthenticatedHttpError.is(response)) { |
||||
return new UnauthenticatedHttpError(this.uri, response); |
||||
} |
||||
if (HttpErrorResult.isnt(response)) { |
||||
return new UnexpectedHttpError(this.uri, response); |
||||
} |
||||
|
||||
if (DataResult.is(response)) { |
||||
// Parse Turtle
|
||||
const rawTurtle = await response.text(); |
||||
let loadedDataset; |
||||
try { |
||||
loadedDataset = await parseRdf(rawTurtle, { |
||||
baseIRI: this.uri, |
||||
}); |
||||
} catch (err) { |
||||
return new TurtleFormattingError( |
||||
this.uri, |
||||
err instanceof Error ? err.message : "Failed to parse rdf", |
||||
); |
||||
} |
||||
|
||||
// Start transaction
|
||||
const transactionalDataset = |
||||
this.context.solidLdoDataset.startTransaction(); |
||||
const graphNode = namedNode(this.uri); |
||||
// Destroy all triples that were once a part of this resouce
|
||||
loadedDataset.deleteMatches(undefined, undefined, undefined, graphNode); |
||||
// Add the triples from the fetched item
|
||||
transactionalDataset.addAll(loadedDataset); |
||||
transactionalDataset.commit(); |
||||
return new DataResult(this.uri); |
||||
} else { |
||||
// Load Blob
|
||||
const blob = await response.blob(); |
||||
return new BinaryResult(this.uri, blob); |
||||
} |
||||
} catch (err) { |
||||
return UnexpectedError.fromThrown(this.uri, err); |
||||
} |
||||
} |
||||
|
||||
// // Create Methods
|
||||
// abstract createDataResource(overwrite?: boolean): Promise<DataLeaf | ResourceError>;
|
||||
|
||||
// abstract upload(
|
||||
// blob: Blob,
|
||||
// mimeType: string,
|
||||
// overwrite?: boolean,
|
||||
// ): Promise<BinaryLeaf | ResourceError>;
|
||||
|
||||
// abstract updateData(
|
||||
// changes: DatasetChanges,
|
||||
// ): Promise<DataLeaf | ResourceError>;
|
||||
|
||||
// abstract delete(): Promise<AbsentLeaf | ResourceError>;
|
||||
} |
||||
@ -0,0 +1,9 @@ |
||||
import { RequesterResult } from "./RequesterResult"; |
||||
|
||||
export class AbsentResult extends RequesterResult { |
||||
type = "absent" as const; |
||||
|
||||
static is(response: Response): boolean { |
||||
return response.status === 404; |
||||
} |
||||
} |
||||
@ -0,0 +1,16 @@ |
||||
import { RequesterResult } from "./RequesterResult"; |
||||
|
||||
export class BinaryResult extends RequesterResult { |
||||
type = "binary" as const; |
||||
readonly blob: Blob; |
||||
|
||||
constructor(uri: string, blob: Blob) { |
||||
super(uri); |
||||
this.blob = blob; |
||||
} |
||||
|
||||
static is(response: Response): boolean { |
||||
const contentType = response.headers.get("content-type"); |
||||
return !contentType || contentType !== "text/turtle"; |
||||
} |
||||
} |
||||
@ -0,0 +1,15 @@ |
||||
import { ErrorResult } from "./ErrorResult"; |
||||
import { RequesterResult } from "./RequesterResult"; |
||||
|
||||
export class DataResult extends RequesterResult { |
||||
type = "data" as const; |
||||
|
||||
static is(response: Response): boolean { |
||||
const contentType = response.headers.get("content-type"); |
||||
return !!contentType && contentType === "text/turtle"; |
||||
} |
||||
} |
||||
|
||||
export class TurtleFormattingError extends ErrorResult { |
||||
errorType = "turtleFormatting"; |
||||
} |
||||
@ -0,0 +1,33 @@ |
||||
export abstract class ErrorResult extends Error { |
||||
readonly type = "error" as const; |
||||
readonly uri: string; |
||||
abstract readonly errorType: string; |
||||
|
||||
constructor(uri: string, message?: string) { |
||||
super(message || "An error unkown error was encountered during a request."); |
||||
this.uri = uri; |
||||
} |
||||
} |
||||
|
||||
export class UnexpectedError extends ErrorResult { |
||||
error: Error; |
||||
readonly errorType = "unexpected"; |
||||
|
||||
constructor(uri: string, error: Error) { |
||||
super(uri, error.message); |
||||
this.error = error; |
||||
} |
||||
|
||||
static fromThrown(uri: string, err: unknown) { |
||||
if (err instanceof Error) { |
||||
return new UnexpectedError(uri, err); |
||||
} else if (typeof err === "string") { |
||||
return new UnexpectedError(uri, new Error(err)); |
||||
} else { |
||||
return new UnexpectedError( |
||||
uri, |
||||
new Error(`Error of type ${typeof err} thrown: ${err}`), |
||||
); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,50 @@ |
||||
import { ErrorResult } from "./ErrorResult"; |
||||
|
||||
export abstract class HttpErrorResult extends ErrorResult { |
||||
public readonly status: number; |
||||
public readonly headers: Headers; |
||||
public readonly response: Response; |
||||
|
||||
constructor(uri: string, response: Response, message?: string) { |
||||
super( |
||||
uri, |
||||
message || |
||||
`Request for ${uri} returned ${response.status} (${response.statusText}).`, |
||||
); |
||||
this.status = response.status; |
||||
this.headers = response.headers; |
||||
this.response = response; |
||||
} |
||||
|
||||
async getBodyForDebug(): Promise<string> { |
||||
if (this.response.bodyUsed) { |
||||
return `Could not get body for ${this.uri} that yeilded status ${this.status}. The body stream has already been consumed.`; |
||||
} |
||||
return await this.response.text(); |
||||
} |
||||
|
||||
static isnt(response: Response) { |
||||
return response.status < 200 || response.status >= 300; |
||||
} |
||||
} |
||||
|
||||
export class UnexpectedHttpError extends HttpErrorResult { |
||||
errorType = "unexpectedHttp" as const; |
||||
} |
||||
|
||||
export class UnauthenticatedHttpError extends HttpErrorResult { |
||||
status: 401; |
||||
errorType = "unauthenticated" as const; |
||||
|
||||
static is(response: Response) { |
||||
return response.status === 404; |
||||
} |
||||
} |
||||
|
||||
export class ServerHttpError extends HttpErrorResult { |
||||
errorType = "server" as const; |
||||
|
||||
static is(response: Response) { |
||||
return response.status >= 500 && response.status < 600; |
||||
} |
||||
} |
||||
@ -0,0 +1,7 @@ |
||||
export abstract class RequesterResult { |
||||
readonly uri: string; |
||||
abstract readonly type: string; |
||||
constructor(uri: string) { |
||||
this.uri = uri; |
||||
} |
||||
} |
||||
@ -0,0 +1,48 @@ |
||||
import type { App } from "@solid/community-server"; |
||||
import { LeafRequester } from "../src/requester/LeafRequester"; |
||||
import crossFetch from "cross-fetch"; |
||||
import { |
||||
createApp, |
||||
getSecret, |
||||
refreshToken, |
||||
type ISecretData, |
||||
type ITokenData, |
||||
getAuthenticatedFetch, |
||||
} from "./solidServer.helper"; |
||||
import { buildAuthenticatedFetch } from "@inrupt/solid-client-authn-core"; |
||||
|
||||
describe("Leaf Requester", () => { |
||||
let app: App; |
||||
let authFetch: typeof fetch; |
||||
|
||||
beforeAll(async () => { |
||||
// Start up the server
|
||||
// app = await createApp();
|
||||
// await app.start();
|
||||
|
||||
authFetch = await getAuthenticatedFetch(); |
||||
}); |
||||
|
||||
it("special request", async () => { |
||||
const response = await authFetch( |
||||
"https://solidweb.me/jackson/everything_public/anonexistentfile.json", |
||||
{ |
||||
method: "PUT", |
||||
headers: { "content-type": "application/json+ld" }, |
||||
body: JSON.stringify({ some: "test" }), |
||||
}, |
||||
); |
||||
console.log("STATUS:", response.status); |
||||
console.log("HEADERS:", response.headers); |
||||
console.log("BODY:", await response.text()); |
||||
}); |
||||
|
||||
it("reads", async () => { |
||||
const leafRequester = new LeafRequester( |
||||
"https://solidweb.me/jackson/everything-public/someotherfile.json", |
||||
{ fetch: authFetch }, |
||||
); |
||||
|
||||
await leafRequester.read(); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,106 @@ |
||||
// Taken from https://github.com/comunica/comunica/blob/b237be4265c353a62a876187d9e21e3bc05123a3/engines/query-sparql/test/QuerySparql-solid-test.ts#L9
|
||||
|
||||
import * as path from "path"; |
||||
import type { KeyPair } from "@inrupt/solid-client-authn-core"; |
||||
import { |
||||
buildAuthenticatedFetch, |
||||
createDpopHeader, |
||||
generateDpopKeyPair, |
||||
} from "@inrupt/solid-client-authn-core"; |
||||
import { AppRunner, resolveModulePath } from "@solid/community-server"; |
||||
import "jest-rdf"; |
||||
import fetch from "cross-fetch"; |
||||
|
||||
const config = [ |
||||
{ |
||||
|
||||
}, |
||||
]; |
||||
|
||||
const SERVER_DOMAIN = "https://solidweb.me"; |
||||
|
||||
// Use an increased timeout, since the CSS server takes too much setup time.
|
||||
jest.setTimeout(40_000); |
||||
|
||||
export function createApp() { |
||||
return new AppRunner().create( |
||||
{ |
||||
mainModulePath: resolveModulePath(""), |
||||
typeChecking: false, |
||||
}, |
||||
resolveModulePath("config/default.json"), |
||||
{}, |
||||
{ |
||||
port: 3_001, |
||||
loggingLevel: "off", |
||||
seededPodConfigJson: path.join( |
||||
__dirname, |
||||
"configs", |
||||
"solid-css-seed.json", |
||||
), |
||||
}, |
||||
); |
||||
} |
||||
|
||||
export interface ISecretData { |
||||
id: string; |
||||
secret: string; |
||||
} |
||||
|
||||
// From https://communitysolidserver.github.io/CommunitySolidServer/5.x/usage/client-credentials/
|
||||
export async function getSecret(): Promise<ISecretData> { |
||||
const result = await fetch(`${SERVER_DOMAIN}/idp/credentials/`, { |
||||
method: "POST", |
||||
headers: { "content-type": "application/json" }, |
||||
body: JSON.stringify({ |
||||
email: config[0].email, |
||||
password: config[0].password, |
||||
name: config[0].podName, |
||||
}), |
||||
}); |
||||
const json = await result.json(); |
||||
return json; |
||||
} |
||||
|
||||
export interface ITokenData { |
||||
accessToken: string; |
||||
dpopKey: KeyPair; |
||||
} |
||||
|
||||
// From https://communitysolidserver.github.io/CommunitySolidServer/5.x/usage/client-credentials/
|
||||
export async function refreshToken({ |
||||
id, |
||||
secret, |
||||
}: ISecretData): Promise<ITokenData> { |
||||
const dpopKey = await generateDpopKeyPair(); |
||||
const authString = `${encodeURIComponent(id)}:${encodeURIComponent(secret)}`; |
||||
const tokenUrl = `${SERVER_DOMAIN}/.oidc/token`; |
||||
const accessToken = await fetch(tokenUrl, { |
||||
method: "POST", |
||||
headers: { |
||||
// The header needs to be in base64 encoding.
|
||||
authorization: `Basic ${Buffer.from(authString).toString("base64")}`, |
||||
"content-type": "application/x-www-form-urlencoded", |
||||
dpop: await createDpopHeader(tokenUrl, "POST", dpopKey), |
||||
}, |
||||
body: "grant_type=client_credentials&scope=webid", |
||||
}) |
||||
.then((res) => res.json()) |
||||
.then((res) => res.access_token); |
||||
|
||||
return { accessToken, dpopKey }; |
||||
} |
||||
|
||||
export async function getAuthenticatedFetch() { |
||||
// Generate secret
|
||||
const secret = await getSecret(); |
||||
|
||||
// Get token
|
||||
const token = await refreshToken(secret); |
||||
|
||||
// Build authenticated fetch
|
||||
const authFetch = await buildAuthenticatedFetch(fetch, token.accessToken, { |
||||
dpopKey: token.dpopKey, |
||||
}); |
||||
return authFetch; |
||||
} |
||||
Loading…
Reference in new issue