diff --git a/.changeset/seven-eels-drop.md b/.changeset/seven-eels-drop.md new file mode 100644 index 000000000..450207d20 --- /dev/null +++ b/.changeset/seven-eels-drop.md @@ -0,0 +1,8 @@ +--- +'@getodk/xforms-engine': minor +'@getodk/web-forms': minor +'@getodk/scenario': minor +'@getodk/common': minor +--- + +Support for external secondary instances (XML, CSV, GeoJSON) diff --git a/package.json b/package.json index 312537995..65ed897c7 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@types/eslint": "^9.6.1", "@types/eslint-config-prettier": "^6.11.3", "@types/eslint__js": "^8.42.3", + "@types/geojson": "^7946.0.14", "@types/jsdom": "^21.1.7", "@types/node": "^22.7.2", "@typescript-eslint/eslint-plugin": "^8.7.0", diff --git a/packages/common/src/fixtures/import-glob-helper.ts b/packages/common/src/fixtures/import-glob-helper.ts new file mode 100644 index 000000000..e89553863 --- /dev/null +++ b/packages/common/src/fixtures/import-glob-helper.ts @@ -0,0 +1,60 @@ +import { IS_NODE_RUNTIME } from '../env/detection.ts'; + +interface GlobURLFetchResponse { + text(): Promise; +} + +type Awaitable = Promise | T; + +type FetchGlobURL = (globURL: string) => Awaitable; + +let fetchGlobURL: FetchGlobURL; + +if (IS_NODE_RUNTIME) { + const { readFile } = await import('node:fs/promises'); + + class NodeGlobURLFetchResponse { + readonly fsPath: string; + + constructor(globURL: string) { + this.fsPath = globURL.replace('/@fs/', '/'); + } + + text(): Promise { + return readFile(this.fsPath, 'utf-8'); + } + } + + fetchGlobURL = (globURL) => { + return new NodeGlobURLFetchResponse(globURL); + }; +} else { + fetchGlobURL = fetch; +} + +type ImportMetaGlobURLRecord = Readonly>; + +export type ImportMetaGlobLoader = (this: void) => Promise; + +export type GlobLoaderEntry = readonly [absolutePath: string, loader: ImportMetaGlobLoader]; + +const globLoader = (globURL: string): ImportMetaGlobLoader => { + return async () => { + const response = await fetchGlobURL(globURL); + + return response.text(); + }; +}; + +export const toGlobLoaderEntries = ( + importMeta: ImportMeta, + globObject: ImportMetaGlobURLRecord +): readonly GlobLoaderEntry[] => { + const parentPathURL = new URL('./', importMeta.url); + + return Object.entries(globObject).map(([relativePath, value]) => { + const { pathname: absolutePath } = new URL(relativePath, parentPathURL); + + return [absolutePath, globLoader(value)]; + }); +}; diff --git a/packages/common/src/fixtures/test-scenario/csv-attachment.csv b/packages/common/src/fixtures/test-scenario/csv-attachment.csv new file mode 100644 index 000000000..38d6bcd32 --- /dev/null +++ b/packages/common/src/fixtures/test-scenario/csv-attachment.csv @@ -0,0 +1,3 @@ +item-label,item-value +Y,y +Z,z diff --git a/packages/common/src/fixtures/test-scenario/xml-attachment.xml b/packages/common/src/fixtures/test-scenario/xml-attachment.xml new file mode 100644 index 000000000..517a46b9b --- /dev/null +++ b/packages/common/src/fixtures/test-scenario/xml-attachment.xml @@ -0,0 +1,10 @@ + + + A + a + + + B + b + + diff --git a/packages/common/src/fixtures/xform-attachments.ts b/packages/common/src/fixtures/xform-attachments.ts new file mode 100644 index 000000000..22d06ce27 --- /dev/null +++ b/packages/common/src/fixtures/xform-attachments.ts @@ -0,0 +1,137 @@ +import { UpsertableMap } from '../lib/collections/UpsertableMap.ts'; +import { UnreachableError } from '../lib/error/UnreachableError.ts'; +import type { ImportMetaGlobLoader } from './import-glob-helper.ts'; +import { toGlobLoaderEntries } from './import-glob-helper.ts'; + +/** + * @todo Support Windows paths? + */ +const getFileName = (absolutePath: string): string => { + const fileName = absolutePath.split('/').at(-1); + + if (fileName == null) { + throw new Error(`Failed to get file name for file system path: ${absolutePath}`); + } + + return fileName; +}; + +// prettier-ignore +const xformAttachmentFileExtensions = [ + '.csv', + '.geojson', + '.xml', + '.xml.example', + '.xlsx', +] as const; + +type XFormAttachmentFileExtensions = typeof xformAttachmentFileExtensions; +type XFormAttachmentFileExtension = XFormAttachmentFileExtensions[number]; + +const getFileExtension = (absolutePath: string): XFormAttachmentFileExtension => { + for (const extension of xformAttachmentFileExtensions) { + if (absolutePath.endsWith(extension)) { + return extension; + } + } + + throw new Error(`Unknown file extension for file name: ${getFileName(absolutePath)}`); +}; + +const getParentDirectory = (absolutePath: string): string => { + const fileName = getFileName(absolutePath); + + return absolutePath.slice(0, absolutePath.length - fileName.length - 1); +}; + +const xformAttachmentFixtureLoaderEntries = toGlobLoaderEntries( + import.meta, + import.meta.glob('./*/**/*', { + query: '?url', + import: 'default', + eager: true, + }) +); + +export class XFormAttachmentFixture { + readonly fileName: string; + readonly fileExtension: string; + readonly mimeType: string; + + constructor( + readonly absolutePath: string, + readonly load: ImportMetaGlobLoader + ) { + const fileName = getFileName(absolutePath); + const fileExtension = getFileExtension(fileName); + + this.fileName = fileName; + this.fileExtension = fileExtension; + + switch (fileExtension) { + case '.csv': + this.mimeType = 'text/csv'; + break; + + case '.geojson': + this.mimeType = 'application/geo+json'; + break; + + case '.xml': + case '.xml.example': + this.mimeType = 'text/xml'; + break; + + case '.xlsx': + this.mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + break; + + default: + throw new UnreachableError(fileExtension); + } + } +} + +type XFormAttachmentFixtureEntry = readonly [absolutePath: string, fixture: XFormAttachmentFixture]; + +type XFormAttachmentFixtureEntries = readonly XFormAttachmentFixtureEntry[]; + +const xformAttachmentFixtureEntries: XFormAttachmentFixtureEntries = + xformAttachmentFixtureLoaderEntries.map(([absolutePath, load]) => { + const fixture = new XFormAttachmentFixture(absolutePath, load); + + return [absolutePath, fixture]; + }); + +type XFormAttachmentFixturesByAbsolutePath = ReadonlyMap; + +const buildXFormAttachmentFixturesByAbsolutePath = ( + entries: XFormAttachmentFixtureEntries +): XFormAttachmentFixturesByAbsolutePath => { + return new Map(entries); +}; + +export const xformAttachmentFixturesByPath = buildXFormAttachmentFixturesByAbsolutePath( + xformAttachmentFixtureEntries +); + +type XFormAttachmentFixturesByDirectory = ReadonlyMap; + +const buildXFormAttachmentFixturesByDirectory = ( + entries: XFormAttachmentFixtureEntries +): XFormAttachmentFixturesByDirectory => { + const result = new UpsertableMap(); + + for (const [absolutePath, fixture] of entries) { + const parentDirectory = getParentDirectory(absolutePath); + const subset = result.upsert(parentDirectory, () => []); + + subset.push(fixture); + } + + return result; +}; + +export const xformAttachmentFixturesByDirectory = buildXFormAttachmentFixturesByDirectory( + xformAttachmentFixtureEntries +); diff --git a/packages/common/src/fixtures/xforms.ts b/packages/common/src/fixtures/xforms.ts index 558186049..1c27e1f28 100644 --- a/packages/common/src/fixtures/xforms.ts +++ b/packages/common/src/fixtures/xforms.ts @@ -1,24 +1,40 @@ -import { IS_NODE_RUNTIME } from '../env/detection.ts'; +import { JRResourceService } from '../jr-resources/JRResourceService.ts'; +import type { JRResourceURL } from '../jr-resources/JRResourceURL.ts'; import { UpsertableMap } from '../lib/collections/UpsertableMap.ts'; +import { toGlobLoaderEntries } from './import-glob-helper.ts'; type XFormResourceType = 'local' | 'remote'; +type ResourceServiceFactory = () => JRResourceService; + interface BaseXFormResourceOptions { readonly localPath: string | null; readonly identifier: string | null; readonly category: string | null; + readonly initializeFormAttachmentService?: ResourceServiceFactory; } interface LocalXFormResourceOptions extends BaseXFormResourceOptions { readonly localPath: string; readonly identifier: string; readonly category: string; + readonly initializeFormAttachmentService: ResourceServiceFactory; } interface RemoteXFormResourceOptions extends BaseXFormResourceOptions { readonly category: string | null; readonly localPath: null; readonly identifier: string; + + /** + * @todo Note that {@link RemoteXFormResourceOptions} corresponds to an API + * primarily serving + * {@link https://getodk.org/web-forms-preview/ | Web Forms Preview} + * functionality. In theory, we could allow a mechanism to support form + * attachments in for that use case, but we'd need to design for it. Until + * then, it doesn't make a whole lot of sense to accept arbitrary IO here. + */ + readonly initializeFormAttachmentService?: never; } type XFormResourceOptions = { @@ -71,20 +87,28 @@ const xformURLLoader = (url: URL): LoadXFormXML => { }; }; +const getNoopResourceService: ResourceServiceFactory = () => { + return new JRResourceService(); +}; + export class XFormResource { static forLocalFixture( - importerURL: string, - relativePath: string, - localURL: URL | string, + localPath: string, + resourceURL: URL, loadXML?: LoadXFormXML ): XFormResource<'local'> { - const resourceURL = new URL(localURL, importerURL); - const localPath = new URL(relativePath, importerURL).pathname; - return new XFormResource('local', resourceURL, loadXML ?? xformURLLoader(resourceURL), { category: localFixtureDirectoryCategory(localPath), localPath, identifier: pathToFileName(localPath), + initializeFormAttachmentService: () => { + const service = new JRResourceService(); + const parentPath = localPath.replace(/\/[^/]+$/, ''); + + service.activateFixtures(parentPath, ['file', 'file-csv']); + + return service; + }, }); } @@ -96,6 +120,8 @@ export class XFormResource { const loadXML = xformURLLoader(resourceURL); return new XFormResource('remote', resourceURL, loadXML, { + ...options, + category: options?.category ?? 'other', identifier: options?.identifier ?? extractURLIdentifier(resourceURL), localPath: options?.localPath ?? null, @@ -105,6 +131,7 @@ export class XFormResource { readonly category: string; readonly localPath: XFormResourceOptions['localPath']; readonly identifier: XFormResourceOptions['identifier']; + readonly fetchFormAttachment: (url: JRResourceURL) => Promise; private constructor( readonly resourceType: Type, @@ -115,37 +142,36 @@ export class XFormResource { this.category = options.category ?? 'other'; this.localPath = options.localPath; this.identifier = options.identifier; - } -} -export type XFormFixture = XFormResource<'local'>; + const initializeFormAttachmentService = + options.initializeFormAttachmentService ?? getNoopResourceService; -const buildXFormFixtures = (): readonly XFormFixture[] => { - if (IS_NODE_RUNTIME) { - const fixtureXMLByRelativePath = import.meta.glob('./**/*.xml', { - query: '?raw', - import: 'default', - eager: false, - }); + let resourceService: JRResourceService | null = null; - return Object.entries(fixtureXMLByRelativePath).map(([path, loadXML]) => { - const localURL = new URL(path, import.meta.url); - const fixture = XFormResource.forLocalFixture(import.meta.url, path, localURL, loadXML); + this.fetchFormAttachment = (url) => { + resourceService = resourceService ?? initializeFormAttachmentService(); - return fixture; - }); + return resourceService.handleRequest(url); + }; } +} - const fixtureURLByRelativePath = import.meta.glob('./**/*.xml', { +const xformFixtureLoaderEntries = toGlobLoaderEntries( + import.meta, + import.meta.glob('./**/*.xml', { query: '?url', import: 'default', eager: true, - }); + }) +); - return Object.entries(fixtureURLByRelativePath).map(([path, url]) => { - const fixture = XFormResource.forLocalFixture(import.meta.url, path, url); +export type XFormFixture = XFormResource<'local'>; + +const buildXFormFixtures = (): readonly XFormFixture[] => { + return xformFixtureLoaderEntries.map(([path, loadXML]) => { + const resourceURL = new URL(path, SELF_URL); - return fixture; + return XFormResource.forLocalFixture(path, resourceURL, loadXML); }); }; diff --git a/packages/common/src/jr-resources/JRResource.ts b/packages/common/src/jr-resources/JRResource.ts new file mode 100644 index 000000000..cd2c71936 --- /dev/null +++ b/packages/common/src/jr-resources/JRResource.ts @@ -0,0 +1,39 @@ +import type { XFormAttachmentFixture } from '../fixtures/xform-attachments.ts'; +import { JRResourceURL } from './JRResourceURL.ts'; + +type JRResourceLoader = (this: void) => Promise; + +export interface JRResourceSource { + readonly url: JRResourceURL; + readonly fileName: string; + readonly mimeType: string; + readonly load: JRResourceLoader; +} + +export class JRResource { + static fromFormAttachmentFixture(category: string, fixture: XFormAttachmentFixture): JRResource { + const { fileName, mimeType, load } = fixture; + const url = JRResourceURL.create(category, fileName); + + return new JRResource({ + url, + fileName, + mimeType, + load, + }); + } + + readonly url: JRResourceURL; + readonly fileName: string; + readonly mimeType: string; + readonly load: JRResourceLoader; + + constructor(source: JRResourceSource) { + const { url, fileName, mimeType, load } = source; + + this.url = url; + this.fileName = fileName; + this.mimeType = mimeType; + this.load = load; + } +} diff --git a/packages/common/src/jr-resources/JRResourceService.ts b/packages/common/src/jr-resources/JRResourceService.ts new file mode 100644 index 000000000..7297f9a73 --- /dev/null +++ b/packages/common/src/jr-resources/JRResourceService.ts @@ -0,0 +1,108 @@ +import { xformAttachmentFixturesByDirectory } from '../fixtures/xform-attachments.ts'; +import { JRResource } from './JRResource.ts'; +import type { JRResourceURLString } from './JRResourceURL.ts'; +import { JRResourceURL } from './JRResourceURL.ts'; + +export interface InlineFixtureMetadata { + readonly url: JRResourceURLString; + readonly fileName: string; + readonly mimeType: string; +} + +class JRResourceServiceRegistry extends Map {} + +interface ActivateFixturesOptions { + readonly suppressMissingFixturesDirectoryWarning?: boolean; +} + +export class JRResourceService { + readonly resources = new JRResourceServiceRegistry(); + + readonly handleRequest = async (url: JRResourceURL | JRResourceURLString): Promise => { + let resourceKey: JRResourceURLString; + + if (typeof url === 'string') { + resourceKey = url; + } else { + resourceKey = url.href; + } + + const resource = this.resources.get(resourceKey); + + if (resource == null) { + return new Response('Not found', { + status: 404, + headers: { 'Content-Type': 'text/plain' }, + }); + } + + const { load, mimeType } = resource; + const body = await load(); + + return new Response(body, { + status: 200, + headers: { 'Content-Type': mimeType }, + }); + }; + + private setRegisteredResourceState(resource: JRResource) { + const url = resource.url.href; + + if (this.resources.has(url)) { + throw new Error(`Resource already registered for URL: ${url}`); + } + + this.resources.set(url, resource); + } + + activateFixtures( + fixtureDirectory: string, + categories: readonly string[], + options?: ActivateFixturesOptions + ): void { + this.reset(); + + try { + for (const category of categories) { + const fixtures = xformAttachmentFixturesByDirectory.get(fixtureDirectory); + + if (fixtures == null) { + if (options?.suppressMissingFixturesDirectoryWarning !== true) { + // eslint-disable-next-line no-console + console.warn(`No form attachments in directory: ${fixtureDirectory}`); + } + + continue; + } + + for (const fixture of fixtures) { + const resource = JRResource.fromFormAttachmentFixture(category, fixture); + + this.setRegisteredResourceState(resource); + } + } + } catch (error) { + // eslint-disable-next-line no-console + console.error('Error occurred during resource state setup:', error); + + this.reset(); + } + } + + activateResource(metadata: InlineFixtureMetadata, data: string): void { + const url = JRResourceURL.from(metadata.url); + const load = () => Promise.resolve(data); + const resource = new JRResource({ + url, + fileName: metadata.fileName, + mimeType: metadata.mimeType, + load, + }); + + this.setRegisteredResourceState(resource); + } + + reset(): void { + this.resources.clear(); + } +} diff --git a/packages/common/src/jr-resources/JRResourceURL.ts b/packages/common/src/jr-resources/JRResourceURL.ts new file mode 100644 index 000000000..32597c0e4 --- /dev/null +++ b/packages/common/src/jr-resources/JRResourceURL.ts @@ -0,0 +1,46 @@ +const JR_RESOURCE_URL_PROTOCOL = 'jr:'; +type JRResourceURLProtocol = typeof JR_RESOURCE_URL_PROTOCOL; + +export type JRResourceURLString = `${JRResourceURLProtocol}${string}`; + +interface ValidatedJRResourceURL extends URL { + readonly protocol: JRResourceURLProtocol; + readonly href: JRResourceURLString; +} + +type ValidateJRResourceURL = (url: URL) => asserts url is ValidatedJRResourceURL; + +const validateJRResourceURL: ValidateJRResourceURL = (url) => { + if (import.meta.env.DEV) { + const { protocol, href } = url; + + if (protocol !== JR_RESOURCE_URL_PROTOCOL || !href.startsWith(JR_RESOURCE_URL_PROTOCOL)) { + throw new Error(`Invalid JRResoruceURL: ${url}`); + } + } +}; + +export class JRResourceURL extends URL { + static create(category: string, fileName: string): JRResourceURL { + return new this(`jr://${category}/${fileName}`); + } + + static from(url: JRResourceURLString): JRResourceURL { + return new this(url); + } + + static isJRResourceReference(reference: string | null): reference is JRResourceURLString { + return reference?.startsWith(JR_RESOURCE_URL_PROTOCOL) ?? false; + } + + declare readonly protocol: JRResourceURLProtocol; + declare readonly href: JRResourceURLString; + + private constructor(url: JRResourceURL); + private constructor(url: JRResourceURLString); + private constructor(url: URL | string) { + super(url); + + validateJRResourceURL(this); + } +} diff --git a/packages/common/src/lib/type-assertions/assertUnknownObject.ts b/packages/common/src/lib/type-assertions/assertUnknownObject.ts index 24692e6d9..a7ac3c461 100644 --- a/packages/common/src/lib/type-assertions/assertUnknownObject.ts +++ b/packages/common/src/lib/type-assertions/assertUnknownObject.ts @@ -1,4 +1,4 @@ -type UnknownObject = Record; +export type UnknownObject = Record; type AssertUnknownObject = (value: unknown) => asserts value is UnknownObject; diff --git a/packages/scenario/src/client/init.ts b/packages/scenario/src/client/init.ts index 03666f3e7..95070bf24 100644 --- a/packages/scenario/src/client/init.ts +++ b/packages/scenario/src/client/init.ts @@ -1,3 +1,4 @@ +import type { JRResourceService } from '@getodk/common/jr-resources/JRResourceService.ts'; import type { XFormsElement } from '@getodk/common/test/fixtures/xform-dsl/XFormsElement.ts'; import type { EngineConfig, @@ -8,6 +9,7 @@ import type { import { initializeForm } from '@getodk/xforms-engine'; import type { Owner } from 'solid-js'; import { createRoot, getOwner, runWithOwner } from 'solid-js'; +import type { MissingResourceBehavior } from '../../../xforms-engine/dist/client/constants'; import { FormDefinitionResource } from '../jr/resource/FormDefinitionResource.ts'; /** @@ -48,16 +50,18 @@ export const getFormResource = async ( * @todo Currently we stub resource fetching. We can address this as needed * while we port existing tests and/or add new ones which require it. */ -const fetchResourceStub: typeof fetch = () => { - throw new Error('TODO: resource fetching not implemented'); +const fetchFormDefinitionStub: typeof fetch = () => { + throw new Error('TODO: fetching form definition not implemented'); }; export interface InitializeTestFormOptions { + readonly resourceService: JRResourceService; + readonly missingResourceBehavior: MissingResourceBehavior; readonly stateFactory: OpaqueReactiveObjectFactory; } const defaultConfig = { - fetchResource: fetchResourceStub, + fetchFormDefinition: fetchFormDefinitionStub, } as const satisfies Omit; interface InitializedTestForm { @@ -82,6 +86,8 @@ export const initializeTestForm = async ( return initializeForm(formResource, { config: { ...defaultConfig, + fetchFormAttachment: options.resourceService.handleRequest, + missingResourceBehavior: options.missingResourceBehavior, stateFactory: options.stateFactory, }, }); diff --git a/packages/scenario/src/jr/Scenario.ts b/packages/scenario/src/jr/Scenario.ts index 4cc3b9613..365872692 100644 --- a/packages/scenario/src/jr/Scenario.ts +++ b/packages/scenario/src/jr/Scenario.ts @@ -11,6 +11,7 @@ import type { SubmissionOptions, SubmissionResult, } from '@getodk/xforms-engine'; +import { constants as ENGINE_CONSTANTS } from '@getodk/xforms-engine'; import type { Accessor, Setter } from 'solid-js'; import { createMemo, createSignal, runWithOwner } from 'solid-js'; import { afterEach, expect } from 'vitest'; @@ -24,6 +25,7 @@ import { getClosestRepeatRange, getNodeForReference } from '../client/traversal. import { ImplementationPendingError } from '../error/ImplementationPendingError.ts'; import { UnclearApplicabilityError } from '../error/UnclearApplicabilityError.ts'; import type { ReactiveScenario } from '../reactive/ReactiveScenario.ts'; +import { SharedJRResourceService } from '../resources/SharedJRResourceService.ts'; import type { JRFormEntryCaption } from './caption/JRFormEntryCaption.ts'; import type { BeginningOfFormEvent } from './event/BeginningOfFormEvent.ts'; import type { EndOfFormEvent } from './event/EndOfFormEvent.ts'; @@ -85,9 +87,8 @@ const isFormFileName = (value: FormDefinitionResource | string): value is FormFi // prettier-ignore type ScenarioStaticInitParameters = - | readonly [formFileName: FormFileName] - | readonly [formName: string, form: XFormsElement] + | readonly [formName: string, form: XFormsElement, overrideOptions?: Partial] | readonly [resource: FormDefinitionResource]; interface AssertCurrentReferenceOptions { @@ -150,9 +151,17 @@ export class Scenario { /** * To be overridden, e.g. by {@link ReactiveScenario}. */ - // eslint-disable-next-line @typescript-eslint/class-literal-property-style - static get initializeTestFormOptions(): InitializeTestFormOptions | null { - return null; + + static getInitializeTestFormOptions( + overrideOptions?: Partial + ): InitializeTestFormOptions { + return { + resourceService: overrideOptions?.resourceService ?? SharedJRResourceService.init(), + missingResourceBehavior: + overrideOptions?.missingResourceBehavior ?? + ENGINE_CONSTANTS.MISSING_RESOURCE_BEHAVIOR.DEFAULT, + stateFactory: overrideOptions?.stateFactory ?? nonReactiveIdentityStateFactory, + }; } static async init( @@ -161,6 +170,7 @@ export class Scenario { ): Promise { let resource: TestFormResource; let formName: string; + let options: InitializeTestFormOptions; if (isFormFileName(args[0])) { return this.init(r(args[0])); @@ -168,18 +178,15 @@ export class Scenario { const [pathResource] = args; resource = pathResource; formName = pathResource.formName; + options = this.getInitializeTestFormOptions(); } else { - const [name, form] = args; + const [name, form, overrideOptions] = args; formName = name; resource = form; + options = this.getInitializeTestFormOptions(overrideOptions); } - const options: InitializeTestFormOptions = { - stateFactory: nonReactiveIdentityStateFactory, - ...this.initializeTestFormOptions, - }; - const { dispose, owner, instanceRoot } = await initializeTestForm(resource, options); return runWithOwner(owner, () => { diff --git a/packages/scenario/src/jr/reference/ReferenceManagerTestUtils.ts b/packages/scenario/src/jr/reference/ReferenceManagerTestUtils.ts index 1f93b4433..d04a06889 100644 --- a/packages/scenario/src/jr/reference/ReferenceManagerTestUtils.ts +++ b/packages/scenario/src/jr/reference/ReferenceManagerTestUtils.ts @@ -1,67 +1,39 @@ -import type { EngineConfig, InitializeFormOptions } from '@getodk/xforms-engine'; -import { afterEach, beforeEach } from 'vitest'; import type { JavaNIOPath } from '../../java/nio/Path.ts'; -import type { TextFileResourcePath } from '../file/TextFileResourcePath.ts'; - -/** - * @todo This is incomplete! It was intended to add preliminary support for - * resource loading consistent with setup in ported JavaRosa tests. Since we - * don't yet have any non-form resource loading logic, we'd have no way of - * exercising the actual functionality. As such, this is currently a sketch of - * what a basis for that might look like in terms of JavaRosa interface - * compatibility, test-scoped setup and teardown. When it does become relevant, - * it will likely intersect with {@link TextFileResourcePath} (or some other - * similar interface to resource fixtures) to service the pertinent resources as - * the engine requests them (via {@link EngineConfig}'s `fetchResource` option). - */ -class ResourceManager { - constructor( - readonly path: JavaNIOPath, - readonly jrResourceBasePaths: readonly string[] - ) {} -} - -let resourceManagers: ResourceManager[] = []; - -beforeEach(() => { - resourceManagers = []; -}); - -afterEach(() => { - resourceManagers = []; -}); +import { SharedJRResourceService } from '../../resources/SharedJRResourceService.ts'; /** * **PORTING NOTES** * - * The name {@link schemes} has been preserved in the signature of this function - * (corresponding to JavaRosa's static method of the same name). It somewhat - * unintuitively **does not** refer to a URL scheme (i.e. `jr:`), but rather the - * first path segment in a `jr:` resource template URL. For instance, if a form - * references a file resource `jr://file/external-data.geojson`, a test may set - * up access to that resource by calling this function and specifying `"files"` - * as a "scheme". - * - * - - - - * - * Exposed as a plain function; addresses pertinent aspects of the semantic - * intent of JavaRosa's same-named static method on the - * `ReferenceManagerTestUtils` class. + * This signature is preserved as a reference to the equivalent JavaRosa test + * setup interface. Some time was spent tracing the actual setup behavior, and + * it was determined (and since confirmed) that ultimately for test purposes the + * intent is to register a set of file system paths which are available for + * resolving fixtures and fixture resources. * - * Significant divergences: + * As such, the actual behavior when calling this function produces the + * following minimal equivalent behavior: * - * 1. Returns `void`, where JavaRosa's equivalent returns a `ReferenceManager` - * (which, per @lognaturel, "nobody likes [...] Don't look :smile:"). This - * appears to be safe for now, as there are no current references to its - * return value. + * 1. When called, any state produced by a prior call is reset. + * 2. The string representation of {@link path} establishes a common base file + * system path for all state produced by the current call. + * 3. For each value in {@link schemes} (naming preserved from JavaRosa), a file + * system path is produced by concatenating that as a subdirectory of that + * common base path. + * 4. Any logic in the active running test will serve fixture resources from the + * set of file system paths produced by the above steps. * - * 2. While also implicitly stateful, the intent is to keep that state scoped as - * clearly as possible to a given test (its state being tracked and cleaned - * up in an `afterEach` controlled locally in this module as well), and as - * minimal as possible to set up the web forms engine's closest semantic - * equivalent (the configuration of `config.fetchResource` in - * {@link InitializeFormOptions}). + * **Implicitly**, the same state is cleared before and after each test, to + * avoid establishing shared state between tests which might cause them to + * become dependent on ordering of test runs. */ export const setUpSimpleReferenceManager = (path: JavaNIOPath, ...schemes: string[]): void => { - resourceManagers.push(new ResourceManager(path, schemes)); + const service = SharedJRResourceService.init(); + + service.activateFixtures(path.toAbsolutePath().toString(), schemes, { + get suppressMissingFixturesDirectoryWarning(): boolean { + const stack = new Error().stack; + + return stack?.includes('configureReferenceManagerIncorrectly') ?? false; + }, + }); }; diff --git a/packages/scenario/src/reactive/ReactiveScenario.ts b/packages/scenario/src/reactive/ReactiveScenario.ts index dd724094e..4765d6c27 100644 --- a/packages/scenario/src/reactive/ReactiveScenario.ts +++ b/packages/scenario/src/reactive/ReactiveScenario.ts @@ -6,10 +6,10 @@ import type { InitializeTestFormOptions } from '../client/init.ts'; import { Scenario, type ScenarioConstructorOptions } from '../jr/Scenario.ts'; export class ReactiveScenario extends Scenario { - static override get initializeTestFormOptions(): InitializeTestFormOptions { - return { + static override getInitializeTestFormOptions(): InitializeTestFormOptions { + return super.getInitializeTestFormOptions({ stateFactory: createMutable, - }; + }); } private readonly testScopedOwner: Owner; diff --git a/packages/scenario/src/resources/SharedJRResourceService.ts b/packages/scenario/src/resources/SharedJRResourceService.ts new file mode 100644 index 000000000..b39d3d035 --- /dev/null +++ b/packages/scenario/src/resources/SharedJRResourceService.ts @@ -0,0 +1,26 @@ +import { JRResourceService } from '@getodk/common/jr-resources/JRResourceService.ts'; +import { afterEach, beforeEach } from 'vitest'; + +let state: SharedJRResourceService | null = null; + +export class SharedJRResourceService extends JRResourceService { + static init(): SharedJRResourceService { + if (state == null) { + state = new this(); + } + + return state; + } + + private constructor() { + super(); + + beforeEach(() => { + this.reset(); + }); + + afterEach(() => { + this.reset(); + }); + } +} diff --git a/packages/scenario/test/actions-events.test.ts b/packages/scenario/test/actions-events.test.ts index 9ebe26d57..3e33cd97e 100644 --- a/packages/scenario/test/actions-events.test.ts +++ b/packages/scenario/test/actions-events.test.ts @@ -1117,8 +1117,8 @@ describe('Actions/Events', () => { * what we might want to change, such as providing a more direct mechanism to * influence the resolution of geolocation data for testing purposes (hint: * it'll probably be configurable in a very similar same way to the - * `fetchResource` engine config option), I also thought it worth mentioning - * these thoughts in anticipation of working on the feature: + * `fetchFormDefinition` engine config option), I also thought it worth + * mentioning these thoughts in anticipation of working on the feature: * * - Any web-native solution will almost certainly be async. * diff --git a/packages/scenario/test/secondary-instances.test.ts b/packages/scenario/test/secondary-instances.test.ts index 7a0c2079c..d9a76517a 100644 --- a/packages/scenario/test/secondary-instances.test.ts +++ b/packages/scenario/test/secondary-instances.test.ts @@ -1,3 +1,6 @@ +import { xformAttachmentFixturesByDirectory } from '@getodk/common/fixtures/xform-attachments.ts'; +import { JRResourceService } from '@getodk/common/jr-resources/JRResourceService.ts'; +import { getBlobText } from '@getodk/common/lib/web-compat/blob.ts'; import { bind, body, @@ -14,7 +17,10 @@ import { t, title, } from '@getodk/common/test/fixtures/xform-dsl/index.ts'; -import { describe, expect, it } from 'vitest'; +import type { PartiallyKnownString } from '@getodk/common/types/string/PartiallyKnownString.ts'; +import { constants as ENGINE_CONSTANTS } from '@getodk/xforms-engine'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { stringAnswer } from '../src/answer/ExpectedStringAnswer.ts'; import { Scenario } from '../src/jr/Scenario.ts'; import { setUpSimpleReferenceManager } from '../src/jr/reference/ReferenceManagerTestUtils.ts'; import { r } from '../src/jr/resource/ResourcePathHelper.ts'; @@ -84,33 +90,25 @@ describe('Secondary instances', () => { * (`answerOf` return value) to be `equalTo(null)`. It seems likely * given the form's shape that the intent is to check that the field is * present and its value is blank, at that point in time. - * - * 3. (HUNCH ONLY!) I'm betting this failure is related to the form's - * `current()` sub-expression (which I doubt is being accounted for in - * dependency analysis, and is therefore failing to establish a - * reactive subscription within the engine). */ // JR: `doNotGetConfused` - it.fails( - "[re]computes separately within each respective repeat instance, when the predicate's dependencies affecting that node change", - async () => { - const scenario = await Scenario.init('repeat-secondary-instance.xml'); + it("[re]computes separately within each respective repeat instance, when the predicate's dependencies affecting that node change", async () => { + const scenario = await Scenario.init('repeat-secondary-instance.xml'); - scenario.createNewRepeat('/data/repeat'); - scenario.createNewRepeat('/data/repeat'); + scenario.createNewRepeat('/data/repeat'); + scenario.createNewRepeat('/data/repeat'); - scenario.answer('/data/repeat[1]/choice', 'a'); + scenario.answer('/data/repeat[1]/choice', 'a'); - expect(scenario.answerOf('/data/repeat[1]/calculate').getValue()).toBe('A'); - // assertThat(scenario.answerOf('/data/repeat[2]/calculate'), equalTo(null)); - expect(scenario.answerOf('/data/repeat[1]/calculate').getValue()).toBe(''); + expect(scenario.answerOf('/data/repeat[1]/calculate').getValue()).toBe('A'); + // assertThat(scenario.answerOf('/data/repeat[2]/calculate'), equalTo(null)); + expect(scenario.answerOf('/data/repeat[2]/calculate').getValue()).toBe(''); - scenario.answer('/data/repeat[2]/choice', 'b'); + scenario.answer('/data/repeat[2]/choice', 'b'); - expect(scenario.answerOf('/data/repeat[1]/calculate').getValue()).toBe('A'); - expect(scenario.answerOf('/data/repeat[2]/calculate').getValue()).toBe('B'); - } - ); + expect(scenario.answerOf('/data/repeat[1]/calculate').getValue()).toBe('A'); + expect(scenario.answerOf('/data/repeat[2]/calculate').getValue()).toBe('B'); + }); }); describe('predicates on different child names', () => { @@ -534,7 +532,7 @@ describe('Secondary instances', () => { * * - Typical `getDisplayText` -> `getValue` */ - it.fails('can be selected', async () => { + it('can be selected', async () => { configureReferenceManagerCorrectly(); const scenario = await Scenario.init(r('external-select-geojson.xml')); @@ -558,11 +556,6 @@ describe('Secondary instances', () => { * Potentially test elsewhere and/or as integration test. */ it.todo('itemsFromExternalSecondaryCSVInstance_ShouldBeAvailableToXPathParser'); - - /** - * **PORTING NOTES** (speculative addition) - */ - it.todo('can select itemset values'); }); }); @@ -622,16 +615,6 @@ describe('Secondary instances', () => { }); describe('CSV secondary instance with header only', () => { - /** - * **PORTING NOTES** - * - * Should probably fail pending feature support. Currently passes because - * this is the expected behavior: - * - * - Without support for external secondary instances (and CSV) - * - * - Without producing an error in their absence - */ it('parses without error', async () => { configureReferenceManagerCorrectly(); @@ -691,24 +674,388 @@ describe('Secondary instances', () => { */ it.todo('dummyNodesInExternalInstanceDeclaration_ShouldBeIgnored'); + /** + * **PORTING NOTES** + * + * This sub-suite has been updated to reflect different semantics and expectations for missing external secondary instances between JavaRosa and Web Forms: + * + * - By default, Web Forms will fail to initialize a form when any of the external secondary instances are missing (i.e. with HTTP 404 semantics). + * + * - By optional configuration, Web Forms may ignore missing external secondary instances, treating them as blank. + */ describe('//region Missing external file', () => { + // JR: emptyPlaceholderInstanceIsUsed_whenExternalInstanceNotFound + it.fails( + '[uses an] empty placeholder [~~]is used[~~] when [referenced] external instance [is] not found', + async () => { + configureReferenceManagerIncorrectly(); + + const scenario = await Scenario.init('external-select-csv.xml'); + + expect(scenario.choicesOf('/data/first').size()).toBe(0); + } + ); + /** * **PORTING NOTES** * - * Should probably fail pending feature support. Currently passes because - * this is the expected behavior: - * - * - Without support for external secondary instances (and CSV) - * - * - Without producing an error in their absence + * Supplemental, exercises configured override of default missing resource + * behavior. */ - it('[uses an] empty placeholder [~~]is used[~~] when [referenced] external instance [is] not found', async () => { + it('uses an empty/blank placeholder when not found, and when overriding configuration is specified', async () => { configureReferenceManagerIncorrectly(); - const scenario = await Scenario.init('external-select-csv.xml'); + const scenario = await Scenario.init( + 'Missing resource treated as blank', + // prettier-ignore + html( + head( + title('Missing resource treated as blank'), + model( + mainInstance( + t('data id="missing-resource-treated-as-blank"', + t('first'))), + + t('instance id="external-csv" src="jr://file-csv/missing.csv"'), + + bind('/data/first').type('string') + ) + ), + body( + select1Dynamic( + '/data/first', + "instance('external-csv')/root/item" + ) + ) + ), + { + missingResourceBehavior: ENGINE_CONSTANTS.MISSING_RESOURCE_BEHAVIOR.BLANK, + } + ); expect(scenario.choicesOf('/data/first').size()).toBe(0); }); }); }); + + describe('basic external secondary instance support', () => { + const xmlAttachmentFileName = 'xml-attachment.xml'; + const xmlAttachmentURL = `jr://file/${xmlAttachmentFileName}` as const; + const csvAttachmentFileName = 'csv-attachment.csv'; + const csvAttachmentURL = `jr://file/${csvAttachmentFileName}` as const; + const formTitle = 'External secondary instance (XML and CSV)'; + const formDefinition = html( + head( + title(formTitle), + model( + // prettier-ignore + mainInstance( + t('data id="external-secondary-instance-xml-csv"', + t('first'), + t('second'))), + + t(`instance id="external-xml" src="${xmlAttachmentURL}"`), + t(`instance id="external-csv" src="${csvAttachmentURL}"`), + + bind('/data/first').type('string'), + bind('/data/second').type('string') + ) + ), + body( + select1Dynamic( + '/data/first', + "instance('external-xml')/instance-root/instance-item", + 'item-value', + 'item-label' + ), + select1Dynamic( + '/data/second', + "instance('external-csv')/root/item", + 'item-value', + 'item-label' + ) + ) + ); + + const activateFixtures = () => { + resourceService.activateFixtures(fixturesDirectory, ['file', 'file-csv']); + }; + + let fixturesDirectory: string; + let resourceService: JRResourceService; + + beforeEach(() => { + const scenarioFixturesDirectory = Array.from(xformAttachmentFixturesByDirectory.keys()).find( + (key) => { + return key.endsWith('/test-scenario'); + } + ); + + if (scenarioFixturesDirectory == null) { + throw new Error(`Failed to get file system path for fixtures directory: "test-scenario"`); + } + + fixturesDirectory = scenarioFixturesDirectory; + + resourceService = new JRResourceService(); + }); + + afterEach(() => { + resourceService.reset(); + }); + + it('supports external secondary instances (XML, file system fixture)', async () => { + activateFixtures(); + + const scenario = await Scenario.init(formTitle, formDefinition, { + resourceService, + }); + + scenario.answer('/data/first', 'a'); + + expect(scenario.answerOf('/data/first')).toEqualAnswer(stringAnswer('a')); + }); + + it('supports external secondary instances (CSV, file system fixture)', async () => { + activateFixtures(); + + const scenario = await Scenario.init(formTitle, formDefinition, { + resourceService, + }); + + scenario.answer('/data/second', 'y'); + + expect(scenario.answerOf('/data/second')).toEqualAnswer(stringAnswer('y')); + }); + }); + + describe('CSV parsing', () => { + const BOM = '\ufeff'; + type BOM = typeof BOM; + + // prettier-ignore + type ColumnDelimiter = + | ',' + | ';' + | '\t' + | '|'; + + // prettier-ignore + type RowDelimiter = + | '\n' + | '\r' + | '\r\n'; + + type ExpectedFailure = 'parse' | 'select-value'; + + interface CSVCase { + readonly description: string; + + /** @default ',' */ + readonly columnDelimiter?: PartiallyKnownString; + + /** @default '\n' */ + readonly rowDelimiter?: PartiallyKnownString; + + /** @default '' */ + readonly bom?: BOM | ''; + + /** @default 0 */ + readonly columnPadding?: number; + + /** @default null */ + readonly expectedFailure?: ExpectedFailure | null; + + /** @default null */ + readonly surprisingSuccessWarning?: string | null; + } + + const csvCases: readonly CSVCase[] = [ + { + description: 'BOM is not treated as part of first column header', + bom: BOM, + }, + { + description: 'column delimiter: semicolon', + columnDelimiter: ';', + }, + { + description: 'column delimiter: tab', + columnDelimiter: '\t', + }, + { + description: 'column delimiter: pipe', + columnDelimiter: '|', + }, + { + description: 'unsupported column delimiter: $', + columnDelimiter: '$', + expectedFailure: 'parse', + }, + { + description: 'row delimiter: LF', + rowDelimiter: '\n', + }, + { + description: 'row delimiter: CR', + rowDelimiter: '\r', + }, + { + description: 'row delimiter: CRLF', + rowDelimiter: '\r\n', + }, + { + description: 'unsupported row delimiter: LFLF', + rowDelimiter: `\n\n`, + expectedFailure: 'parse', + }, + + { + description: 'somewhat surprisingly supported row delimiter: LFCR', + rowDelimiter: `\n\r`, + surprisingSuccessWarning: + "LFCR is not an expected line separator in any known-common usage. It's surprising that Papaparse does not fail parsing this case, at least parsing rows!", + }, + + { + description: 'whitespace padding around column delimiter is not ignored (by default)', + columnDelimiter: ',', + columnPadding: 1, + expectedFailure: 'select-value', + }, + ]; + + // Note: this isn't set up with `describe.each` because it would create a superfluous outer description where the inner description must be applied with `it` (to perform async setup) + csvCases.forEach( + ({ + description, + columnDelimiter = ',', + rowDelimiter = '\n', + bom = '', + columnPadding = 0, + expectedFailure = null, + surprisingSuccessWarning = null, + }) => { + const LOWER_ALPHA_ASCII_LETTER_COUNT = 26; + const lowerAlphaASCIILetters = Array.from( + { + length: LOWER_ALPHA_ASCII_LETTER_COUNT, + }, + (_, i) => { + return String.fromCharCode(i + 97); + } + ); + + type CSVRow = readonly [itemLabel: string, itemValue: string]; + + const rows: readonly CSVRow[] = [ + ['item-label', 'item-value'], + + ...lowerAlphaASCIILetters.map((letter): CSVRow => { + return [letter.toUpperCase(), letter]; + }), + ]; + const baseCSVFixture = rows + .map((row) => { + const padding = ' '.repeat(columnPadding); + const delimiter = `${padding}${columnDelimiter}${padding}`; + + return row.join(delimiter); + }) + .join(rowDelimiter); + + const csvAttachmentFileName = 'csv-attachment.csv'; + const csvAttachmentURL = `jr://file/${csvAttachmentFileName}` as const; + const formTitle = 'External secondary instance (CSV)'; + const formDefinition = html( + head( + title(formTitle), + model( + // prettier-ignore + mainInstance( + t('data id="external-secondary-instance-csv"', + t('letter'))), + + t(`instance id="external-csv" src="${csvAttachmentURL}"`), + + bind('/data/letter').type('string') + ) + ), + body( + select1Dynamic( + '/data/letter', + "instance('external-csv')/root/item", + 'item-value', + 'item-label' + ) + ) + ); + + let resourceService: JRResourceService; + + beforeEach(() => { + resourceService = new JRResourceService(); + }); + + afterEach(() => { + resourceService.reset(); + }); + + it(description, async () => { + let csvFixture: string; + + if (bom === '') { + csvFixture = baseCSVFixture; + } else { + const blob = new Blob([bom, baseCSVFixture]); + + csvFixture = await getBlobText(blob); + } + + resourceService.activateResource( + { + url: csvAttachmentURL, + fileName: csvAttachmentFileName, + mimeType: 'text/csv', + }, + csvFixture + ); + + const letterIndex = Math.floor(Math.random() * LOWER_ALPHA_ASCII_LETTER_COUNT); + const letter = lowerAlphaASCIILetters[letterIndex]!; + + const initScenario = async (): Promise => { + return await Scenario.init(formTitle, formDefinition, { + resourceService, + }); + }; + + if (expectedFailure === 'parse') { + const initParseFailure = async () => { + await initScenario(); + }; + + await expect(initParseFailure).rejects.toThrowError(); + + return; + } + + if (surprisingSuccessWarning != null) { + // eslint-disable-next-line no-console + console.warn(surprisingSuccessWarning); + } + + const scenario = await initScenario(); + + scenario.answer('/data/letter', letter); + + if (expectedFailure === 'select-value') { + expect(scenario.answerOf('/data/letter')).toEqualAnswer(stringAnswer('')); + } else { + expect(scenario.answerOf('/data/letter')).toEqualAnswer(stringAnswer(letter)); + } + }); + } + ); + }); }); diff --git a/packages/scenario/test/serialization.test.ts b/packages/scenario/test/serialization.test.ts index 54e16e1ed..c4e5536d0 100644 --- a/packages/scenario/test/serialization.test.ts +++ b/packages/scenario/test/serialization.test.ts @@ -175,10 +175,11 @@ describe('ExternalSecondaryInstanceParseTest.java', () => { * - Insofar as we may find ourselves implementing similar logic (albeit * serving other purposes), how can we establish a clear interface * contract around behaviors like this? Should it be more consistent? Does - * our current {@link EngineConfig.fetchResource} option—configurable in - * {@link InitializeFormOptions}—provide enough informational surface area - * to communicate such intent (and allow both clients and engine alike to - * have clarity of that intent at call/handling sites)? + * our current {@link EngineConfig.fetchFormDefinition} + * option—configurable in {@link InitializeFormOptions}—provide enough + * informational surface area to communicate such intent (and allow both + * clients and engine alike to have clarity of that intent at + * call/handling sites)? * * - - - * diff --git a/packages/web-forms/src/components/OdkWebForm.vue b/packages/web-forms/src/components/OdkWebForm.vue index be7730580..5f4a21903 100644 --- a/packages/web-forms/src/components/OdkWebForm.vue +++ b/packages/web-forms/src/components/OdkWebForm.vue @@ -1,5 +1,6 @@