diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 536715f673f..f07a0e09b81 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -359,7 +359,7 @@ export interface Field< emptyValue(instance: BaseDef): any; validate(instance: BaseDef, value: any): void; component(model: Box): BoxComponent; - getter(instance: BaseDef): BaseInstanceType; + getter(instance: BaseDef): BaseInstanceType | undefined; queryableValue(value: any, stack: BaseDef[]): SearchT; handleNotLoadedError( instance: BaseInstanceType, @@ -426,13 +426,16 @@ class ContainsMany return this.cardThunk(); } - getter(instance: BaseDef): BaseInstanceType { + getter(instance: BaseDef): BaseInstanceType | undefined { let deserialized = getDataBucket(instance); entangleWithCardTracking(instance); let maybeNotLoaded = deserialized.get(this.name); // a not loaded error can blow up thru a computed containsMany field that consumes a link if (isNotLoadedValue(maybeNotLoaded)) { lazilyLoadLink(instance as CardDef, this, maybeNotLoaded.reference); + if ((globalThis as any).__lazilyLoadLinks) { + return this.emptyValue(instance) as BaseInstanceType; + } } return getter(instance, this); } @@ -736,13 +739,16 @@ class Contains implements Field { return this.cardThunk(); } - getter(instance: BaseDef): BaseInstanceType { + getter(instance: BaseDef): BaseInstanceType | undefined { let deserialized = getDataBucket(instance); entangleWithCardTracking(instance); let maybeNotLoaded = deserialized.get(this.name); // a not loaded error can blow up thru a computed contains field that consumes a link if (isNotLoadedValue(maybeNotLoaded)) { lazilyLoadLink(instance as CardDef, this, maybeNotLoaded.reference); + if ((globalThis as any).__lazilyLoadLinks) { + return undefined; + } } return getter(instance, this); } @@ -934,13 +940,16 @@ class LinksTo implements Field { return this.cardThunk(); } - getter(instance: CardDef): BaseInstanceType { + getter(instance: CardDef): BaseInstanceType | undefined { let deserialized = getDataBucket(instance); // this establishes that our field should rerender when cardTracking for this card changes entangleWithCardTracking(instance); let maybeNotLoaded = deserialized.get(this.name); if (isNotLoadedValue(maybeNotLoaded)) { lazilyLoadLink(instance, this, maybeNotLoaded.reference); + if ((globalThis as any).__lazilyLoadLinks) { + return undefined; + } } return getter(instance, this); } @@ -1322,7 +1331,7 @@ class LinksToMany value = this.emptyValue(instance); deserialized.set(this.name, value); lazilyLoadLink(instance, this, value.reference, { value }); - return this.emptyValue as BaseInstanceType; + return this.emptyValue(instance) as BaseInstanceType; } // Ensure we have an array - if not, something went wrong during deserialization @@ -1342,6 +1351,31 @@ class LinksToMany } } if (notLoadedRefs.length > 0) { + // Important: we intentionally leave the NotLoadedValue sentinels inside the + // WatchedArray so the lazy loader can swap them out in place once the linked + // cards finish loading. Because the array identity never changes, Glimmer’s + // tracking sees the mutation and re-renders when lazilyLoadLink replaces each + // sentinel with a CardDef instance. Callers should treat these entries as + // placeholders (e.g. check for constructor.getComponent) rather than assuming + // every element is immediately renderable. Ideally the .value refactor can + // iron out this kink. + // TODO + // Codex has offered a couple interim solutions to ease the burden on card + // authors around this: + // We can wrap the guard in a reusable helper/component so card authors don’t + // have to think about the sentinel: + // + // - Helper – export something like `has-card-component` (just checks + // `value?.constructor?.getComponent`) from card-api. Then in templates + // they write: `{{#if (has-card-component card)}}…{{/if}}` or + // `{{#each (filter-loadable cards) as |c|}}`. + // + // - Component – provide a `LoadableCard` component that takes a card instance + // and renders the correct `CardContainer` only when the component is ready; + // otherwise it renders nothing or a skeleton. Card authors use + // `` instead of calling `getComponent` + // themselves. + if (!(globalThis as any).__lazilyLoadLinks) { throw new NotLoaded(instance, notLoadedRefs, this.name); } diff --git a/packages/catalog-realm/catalog-app/components/card-with-hydration.gts b/packages/catalog-realm/catalog-app/components/card-with-hydration.gts index ef8384242aa..61416a54361 100644 --- a/packages/catalog-realm/catalog-app/components/card-with-hydration.gts +++ b/packages/catalog-realm/catalog-app/components/card-with-hydration.gts @@ -7,6 +7,7 @@ import { tracked } from '@glimmer/tracking'; import { type CardContext, type BaseDef, + type BoxComponent, } from 'https://cardstack.com/base/card-api'; import { type PrerenderedCardLike } from '@cardstack/runtime-common'; @@ -41,12 +42,14 @@ export class CardWithHydration extends GlimmerComponent + {{#if Component}} + + {{/if}} {{/let}} {{/if}} {{else if @card.isError}} @@ -101,10 +104,20 @@ export class CardWithHydration extends GlimmerComponent } -function getComponent(cardOrField: BaseDef) { - return cardOrField.constructor.getComponent(cardOrField); +function getComponent(cardOrField: BaseDef | undefined | null): BoxComponent | undefined { + if (!cardOrField) { + return; + } + let constructor = cardOrField.constructor as typeof BaseDef | undefined; + if (typeof constructor?.getComponent !== 'function') { + return; + } + return constructor.getComponent(cardOrField); } -function removeFileExtension(cardUrl: string) { - return cardUrl?.replace(/\.[^/.]+$/, ''); +function removeFileExtension(cardUrl: string | undefined | null) { + if (typeof cardUrl !== 'string') { + return ''; + } + return cardUrl.replace(/\.[^/.]+$/, ''); } diff --git a/packages/catalog-realm/catalog-app/components/cards-display-section.gts b/packages/catalog-realm/catalog-app/components/cards-display-section.gts index 728b94bed66..0c58a2b12fd 100644 --- a/packages/catalog-realm/catalog-app/components/cards-display-section.gts +++ b/packages/catalog-realm/catalog-app/components/cards-display-section.gts @@ -31,16 +31,18 @@ export class CardsIntancesGrid extends GlimmerComponent {
    {{#each @cards key='url' as |card|}} {{#let (getComponent card) as |CardComponent|}} -
  • - - - -
    • + {{#if CardComponent}} +
  • + + + +
    • + {{/if}} {{/let}} {{/each}}
@@ -105,10 +107,19 @@ export class CardsIntancesGrid extends GlimmerComponent { } function getComponent(cardOrField: BaseDef) { - return cardOrField.constructor.getComponent(cardOrField); + if ( + !cardOrField || + typeof (cardOrField.constructor as { getComponent?: unknown })?.getComponent !== 'function' + ) { + return; + } + return (cardOrField.constructor as typeof CardDef).getComponent(cardOrField); } -function removeFileExtension(cardUrl: string) { +function removeFileExtension(cardUrl: string | undefined | null) { + if (typeof cardUrl !== 'string') { + return ''; + } return cardUrl.replace(/\.[^/.]+$/, ''); } diff --git a/packages/catalog-realm/catalog-app/listing/listing.gts b/packages/catalog-realm/catalog-app/listing/listing.gts index c327f46ab99..a19d13c82da 100644 --- a/packages/catalog-realm/catalog-app/listing/listing.gts +++ b/packages/catalog-realm/catalog-app/listing/listing.gts @@ -7,6 +7,7 @@ import { StringField, linksTo, Component, + instanceOf, realmURL, } from 'https://cardstack.com/base/card-api'; import { commandData } from 'https://cardstack.com/base/resources/command-data'; @@ -579,7 +580,9 @@ export class SkillListing extends Listing { function specBreakdown(specs: Spec[]): Record { return specs.reduce( (groupedSpecs, spec) => { - if (!spec) { + if (!spec || !instanceOf(spec, Spec)) { + // During prerender linksToMany may still contain not-loaded placeholders; + // skip until the real Spec instance arrives. return groupedSpecs; } let key = spec.specType ?? 'unknown'; diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index a24fd29e1b9..e44f034a69c 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -18,6 +18,7 @@ import { type PrerenderMeta, type RenderRouteOptions, serializeRenderRouteOptions, + cleanCapturedHTML, } from '@cardstack/runtime-common'; import { readFileAsText as _readFileAsText } from '@cardstack/runtime-common/stream'; import { @@ -497,17 +498,6 @@ function omitOneTimeOptions(options: RenderRouteOptions): RenderRouteOptions { return options; } -function cleanCapturedHTML(html: string): string { - if (!html) { - return html; - } - const emberIdAttr = /\s+id=(?:"ember\d+"|'ember\d+'|ember\d+)(?=[\s>])/g; - const emptyDataAttr = /\s+(data-[A-Za-z0-9:_-]+)=(?:""|''|(?=[\s>]))/g; - let cleaned = html.replace(emberIdAttr, ''); - cleaned = cleaned.replace(emptyDataAttr, ' $1'); - return cleaned; -} - function extractPrerenderError(markup: string): string | undefined { if (!markup.includes('data-prerender-error')) { return undefined; diff --git a/packages/host/app/lib/current-run.ts b/packages/host/app/lib/current-run.ts index 65f7e0b596b..b6925646566 100644 --- a/packages/host/app/lib/current-run.ts +++ b/packages/host/app/lib/current-run.ts @@ -24,6 +24,7 @@ import { jobIdentity, getFieldDefinitions, modulesConsumedInMeta, + cleanCapturedHTML, type ResolvedCodeRef, type Definition, type Batch, @@ -1202,8 +1203,7 @@ export class CurrentRun { } function sanitizeHTML(html: string): string { - // currently this only involves removing auto-generated ember ID's - return html.replace(/\s+id="ember[0-9]+"/g, ''); + return cleanCapturedHTML(html); } function assertURLEndsWithJSON(url: URL): URL { diff --git a/packages/host/app/routes/render.ts b/packages/host/app/routes/render.ts index 5e20ee1d385..a7b3540f35a 100644 --- a/packages/host/app/routes/render.ts +++ b/packages/host/app/routes/render.ts @@ -86,8 +86,8 @@ export default class RenderRoute extends Route { currentURL: this.router.currentURL, }); this.#setAllModelStatuses('unusable'); - (globalThis as any)._lazilyLoadLinks = undefined; - (globalThis as any)._boxelRenderContext = undefined; + (globalThis as any).__lazilyLoadLinks = undefined; + (globalThis as any).__boxelRenderContext = undefined; }; activate() { @@ -98,8 +98,8 @@ export default class RenderRoute extends Route { } deactivate() { - (globalThis as any)._lazilyLoadLinks = undefined; - (globalThis as any)._boxelRenderContext = undefined; + (globalThis as any).__lazilyLoadLinks = undefined; + (globalThis as any).__boxelRenderContext = undefined; (globalThis as any).__renderInstance = undefined; window.removeEventListener('error', this.errorHandler); window.removeEventListener('unhandledrejection', this.errorHandler); @@ -455,10 +455,10 @@ export default class RenderRoute extends Route { try { let cardError: CardError = JSON.parse(error.message); return JSON.stringify( - { + this.#stripLastKnownGoodHtml({ type: 'error', error: cardError, - } as RenderError, + } as RenderError), null, 2, ); @@ -473,15 +473,44 @@ export default class RenderRoute extends Route { } while (current && !id); if (isCardError(error)) { return JSON.stringify( - { type: 'error', error: serializableError(error) }, + this.#stripLastKnownGoodHtml({ + type: 'error', + error: serializableError(error), + }), null, 2, ); } let errorJSONAPI = formattedError(id, error).errors[0]; let errorPayload = errorJsonApiToErrorEntry(errorJSONAPI); - return JSON.stringify(errorPayload, null, 2); + return JSON.stringify( + this.#stripLastKnownGoodHtml(errorPayload), + null, + 2, + ); + } + } + + #stripLastKnownGoodHtml(value: T): T { + if (Array.isArray(value)) { + return value.map((item) => + this.#stripLastKnownGoodHtml(item), + ) as unknown as T; + } + if (value && typeof value === 'object') { + let entries = Object.entries(value).reduce>( + (acc, [key, val]) => { + if (key === 'lastKnownGoodHtml') { + return acc; + } + acc[key] = this.#stripLastKnownGoodHtml(val); + return acc; + }, + {}, + ); + return entries as T; } + return value; } #deriveErrorContext(transition?: Transition): { diff --git a/packages/host/scripts/test-wait-for-servers.sh b/packages/host/scripts/test-wait-for-servers.sh index 03d2ce73ad1..b2665d2f293 100755 --- a/packages/host/scripts/test-wait-for-servers.sh +++ b/packages/host/scripts/test-wait-for-servers.sh @@ -37,7 +37,7 @@ TEST_REALM_READY="$TEST_REALM$READY_PATH" SYNAPSE_URL="http://localhost:8008" SMTP_4_DEV_URL="http://localhost:5001" -NODE_NO_WARNINGS=1 start-server-and-test \ +WAIT_ON_TIMEOUT=1200000 NODE_NO_WARNINGS=1 start-server-and-test \ 'pnpm run wait' \ "$BASE_REALM_READY|$CATALOG_REALM_READY|$NODE_TEST_REALM_READY|$SKILLS_REALM_READY|$TEST_REALM_READY|$SYNAPSE_URL|$SMTP_4_DEV_URL" \ 'ember-test-pre-built' diff --git a/packages/matrix/scripts/test.sh b/packages/matrix/scripts/test.sh index ee927c34dbe..ceee659e4d3 100755 --- a/packages/matrix/scripts/test.sh +++ b/packages/matrix/scripts/test.sh @@ -14,7 +14,7 @@ TEST_REALM_READY="$TEST_REALM$READY_PATH" HOST_PATH="http://127.0.0.1:4200" -start-server-and-test \ +WAIT_ON_TIMEOUT=600000 start-server-and-test \ 'pnpm run wait' \ "$BASE_REALM_READY|$NODE_TEST_REALM_READY|$TEST_REALM_READY" \ "pnpm run start:host-pre-built" \ diff --git a/packages/postgres/pg-queue.ts b/packages/postgres/pg-queue.ts index f13499b4415..c10787999a4 100644 --- a/packages/postgres/pg-queue.ts +++ b/packages/postgres/pg-queue.ts @@ -19,6 +19,7 @@ import { PgAdapter } from './pg-adapter'; import * as Sentry from '@sentry/node'; const log = logger('queue'); +const MAX_JOB_TIMEOUT_SEC = 10 * 60; interface JobsTable { id: number; @@ -234,7 +235,7 @@ export class PgQueueRunner implements QueueRunner { constructor({ adapter, workerId, - maxTimeoutSec = 5 * 60, + maxTimeoutSec = MAX_JOB_TIMEOUT_SEC, priority = 0, }: { adapter: PgAdapter; diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 160e64d60ed..cc910fe1b4f 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -138,6 +138,9 @@ export function buildPrerenderApp( pool, }, }; + if (pool.timedOut) { + log.warn(`render of ${url} timed out`); + } if (response.error) { log.debug( `render of ${url} resulted in error doc:\n${JSON.stringify(response.error, null, 2)}`, diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index ecd4ff557aa..99b863af164 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -31,6 +31,13 @@ import { const log = logger('prerenderer'); const boxelHostURL = process.env.BOXEL_HOST_URL ?? 'http://localhost:4200'; +const CLEAR_CACHE_RETRY_SIGNATURES: readonly (readonly string[])[] = [ + // this is a side effect of glimmer scoped styles moving a DOM node that + // glimmer is tracking. when we go to teardown the component glimmer gets mad + // that a node it was tracking is no longer there. performing a new prerender + // capture with a cleared store/loader cache will workaround this issue. + [`Failed to execute 'removeChild' on 'Node'`, 'NotFoundError'], +]; export class Prerenderer { #browser: Browser | null = null; @@ -303,13 +310,6 @@ export class Prerenderer { timedOut: boolean; }; }> { - this.#nonce++; - log.info( - `prerendering url ${url}, nonce=${this.#nonce} realm=${realm} userId=${userId}`, - ); - log.debug( - `prerendering url ${url} with permissions=${JSON.stringify(permissions)}`, - ); if (this.#stopped) { throw new Error('Prerenderer has been stopped and cannot be used'); } @@ -325,19 +325,6 @@ export class Prerenderer { await prev.catch((e) => { log.debug('Previous prerender in chain failed (continuing):', e); }); // ensure chain continues even after errors - const { page, reused, launchMs } = await this.#getPage(realm); - const poolInfo = { - pageId: this.#pool.get(realm)?.pageId ?? 'unknown', - realm, - reused, - evicted: false, - timedOut: false, - }; - const markTimeout = (err?: RenderError) => { - if (!poolInfo.timedOut && err?.error?.title === 'Render timeout') { - poolInfo.timedOut = true; - } - }; let sessions: { [realm: string]: string } = {}; for (let [realmURL, realmPermissions] of Object.entries( @@ -356,230 +343,360 @@ export class Prerenderer { } let auth = JSON.stringify(sessions); - if (!reused) { - page.evaluateOnNewDocument((auth) => { - localStorage.setItem('boxel-session', auth); - }, auth); - } else { - // Only set immediately when reusing an already-loaded document; on a fresh - // navigation, calling localStorage on about:blank can throw a SecurityError. - await page.evaluate((auth) => { - localStorage.setItem('boxel-session', auth); - }, auth); - } - - let renderStart = Date.now(); - let error: RenderError | undefined; - let shortCircuit = false; - let options = renderOptions ?? {}; - let serializedOptions = serializeRenderRouteOptions(options); - let optionsSegment = encodeURIComponent(serializedOptions); - const captureOptions: CaptureOptions = { - expectedId: url.replace(/\.json$/i, ''), - expectedNonce: String(this.#nonce), - simulateTimeoutMs: opts?.simulateTimeoutMs, - }; - - // We need to render the isolated HTML view first, as the template will pull linked fields. - let result = await withTimeout( - page, - async () => { - if (reused) { - await transitionTo( - page, - 'render.html', - url, - String(this.#nonce), - serializedOptions, - 'isolated', - '0', - ); - } else { - await page.goto( - `${boxelHostURL}/render/${encodeURIComponent(url)}/${this.#nonce}/${optionsSegment}/html/isolated/0`, - ); + let attemptOptions = renderOptions; + let lastResult: + | { + response: RenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; } - return await captureResult(page, 'innerHTML', captureOptions); - }, - opts?.timeoutMs, - ); - let isolatedHTML: string | null = null; - if (isRenderError(result)) { - error = result; - markTimeout(error); - let evicted = await this.#maybeEvict( + | undefined; + for (let attempt = 0; attempt < 2; attempt++) { + let result = await this.#prerenderAttempt({ realm, - 'isolated render', - result as RenderError, - ); - if (evicted) { - poolInfo.evicted = true; - shortCircuit = true; + url, + userId, + permissions, + auth, + opts, + renderOptions: attemptOptions, + }); + lastResult = result; + + let retrySignature = this.#shouldRetryWithClearCache(result.response); + let isClearCacheAttempt = attemptOptions?.clearCache === true; + + if (!isClearCacheAttempt && retrySignature) { + log.warn( + `retrying prerender for ${url} with clearCache due to error signature: ${retrySignature.join( + ' | ', + )}`, + ); + attemptOptions = { + ...(attemptOptions ?? {}), + clearCache: true, + }; + continue; } - } else { - let capture = result as RenderCapture; - if (capture.status === 'ready') { - isolatedHTML = capture.value; - } else { - let capErr = this.#captureToError(capture); - if (!error && capErr) { - error = capErr; - } - markTimeout(capErr); - let evicted = await this.#maybeEvict( - realm, - 'isolated render', - capErr, + + if (isClearCacheAttempt && retrySignature && result.response.error) { + log.warn( + `prerender retry with clearCache did not resolve error signature ${retrySignature.join( + ' | ', + )} for ${url}`, ); - if (evicted) { - poolInfo.evicted = true; - shortCircuit = true; - } } + + return result; } + if (lastResult) { + if (lastResult.response.error) { + log.error( + `prerender attempts exhausted for ${url} in realm ${realm}, returning last error response`, + ); + } + return lastResult; + } + throw new Error(`prerender attempts exhausted for ${url}`); + } finally { + deferred.fulfill(); + } + } - if (shortCircuit) { - let meta: PrerenderMeta = { - serialized: null, - searchDoc: null, - displayNames: null, - deps: null, - types: null, - }; - return { - response: { - ...meta, - ...(error ? { error } : {}), - iconHTML: null, - isolatedHTML, - atomHTML: null, - embeddedHTML: null, - fittedHTML: null, - }, - timings: { launchMs, renderMs: Date.now() - renderStart }, - pool: poolInfo, - }; + async #prerenderAttempt({ + realm, + url, + userId, + permissions, + auth, + opts, + renderOptions, + }: { + realm: string; + url: string; + userId: string; + permissions: RealmPermissions; + auth: string; + opts?: { timeoutMs?: number; simulateTimeoutMs?: number }; + renderOptions?: RenderRouteOptions; + }): Promise<{ + response: RenderResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + realm: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }> { + this.#nonce++; + log.info( + `prerendering url ${url}, nonce=${this.#nonce} realm=${realm} userId=${userId}`, + ); + log.debug( + `prerendering url ${url} with permissions=${JSON.stringify(permissions)}`, + ); + + const { page, reused, launchMs } = await this.#getPage(realm); + const poolInfo = { + pageId: this.#pool.get(realm)?.pageId ?? 'unknown', + realm, + reused, + evicted: false, + timedOut: false, + }; + const markTimeout = (err?: RenderError) => { + if (!poolInfo.timedOut && err?.error?.title === 'Render timeout') { + poolInfo.timedOut = true; } + }; - // TODO consider breaking out rendering search doc into its own route so - // that we can fully understand all the linked fields that are used in all - // the html formats and generate a search doc that is well populated. Right - // now we only consider linked fields used in the isolated template. - let metaMaybeError = await withTimeout( - page, - () => renderMeta(page, captureOptions), - opts?.timeoutMs, + if (!reused) { + page.evaluateOnNewDocument((auth) => { + localStorage.setItem('boxel-session', auth); + }, auth); + } else { + // Only set immediately when reusing an already-loaded document; on a fresh + // navigation, calling localStorage on about:blank can throw a SecurityError. + await page.evaluate((auth) => { + localStorage.setItem('boxel-session', auth); + }, auth); + } + + let renderStart = Date.now(); + let error: RenderError | undefined; + let shortCircuit = false; + let options = renderOptions ?? {}; + let serializedOptions = serializeRenderRouteOptions(options); + let optionsSegment = encodeURIComponent(serializedOptions); + const captureOptions: CaptureOptions = { + expectedId: url.replace(/\.json$/i, ''), + expectedNonce: String(this.#nonce), + simulateTimeoutMs: opts?.simulateTimeoutMs, + }; + + // We need to render the isolated HTML view first, as the template will pull linked fields. + let result = await withTimeout( + page, + async () => { + if (reused) { + await transitionTo( + page, + 'render.html', + url, + String(this.#nonce), + serializedOptions, + 'isolated', + '0', + ); + } else { + await page.goto( + `${boxelHostURL}/render/${encodeURIComponent(url)}/${this.#nonce}/${optionsSegment}/html/isolated/0`, + ); + } + return await captureResult(page, 'innerHTML', captureOptions); + }, + opts?.timeoutMs, + ); + let isolatedHTML: string | null = null; + if (isRenderError(result)) { + error = result; + markTimeout(error); + let evicted = await this.#maybeEvict( + realm, + 'isolated render', + result as RenderError, ); - // TODO also consider introducing a mechanism in the API to track and reset - // field usage for an instance recursively so that the depth that an - // instance is loaded from a different rendering context in the same realm - // doesn't elide fields that this rendering context cares about. in that - // manner we can get a complete picture of how to build the search doc's linked - // fields for each rendering context. - let meta: PrerenderMeta; - if (isRenderError(metaMaybeError)) { - markTimeout(metaMaybeError as RenderError); - if ( - await this.#maybeEvict( - realm, - 'render.meta', - metaMaybeError as RenderError, - ) - ) { + if (evicted) { + poolInfo.evicted = true; + shortCircuit = true; + } + } else { + let capture = result as RenderCapture; + if (capture.status === 'ready') { + isolatedHTML = capture.value; + } else { + let capErr = this.#captureToError(capture); + if (!error && capErr) { + error = capErr; + } + markTimeout(capErr); + let evicted = await this.#maybeEvict(realm, 'isolated render', capErr); + if (evicted) { poolInfo.evicted = true; shortCircuit = true; } - error = error ?? (metaMaybeError as RenderError); - markTimeout(error); - meta = { - serialized: null, - searchDoc: null, - displayNames: null, - deps: null, - types: null, - }; - } else { - meta = metaMaybeError; } - let atomHTML: string | null = null, - iconHTML: string | null = null, - embeddedHTML: Record | null = null, - fittedHTML: Record | null = null; - if (!shortCircuit && meta.types) { - // Render sequentially and short-circuit on unusable page/timeout - const steps: Array<{ - name: string; - cb: () => Promise | RenderError>; - assign: (value: string | Record) => void; - }> = [ - { - name: 'fitted render', - cb: () => - renderAncestors(page, 'fitted', meta.types!, captureOptions), - assign: (v: string | Record) => { - fittedHTML = v as Record; - }, + } + + if (shortCircuit) { + let meta: PrerenderMeta = { + serialized: null, + searchDoc: null, + displayNames: null, + deps: null, + types: null, + }; + return { + response: { + ...meta, + ...(error ? { error } : {}), + iconHTML: null, + isolatedHTML, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + }, + timings: { launchMs, renderMs: Date.now() - renderStart }, + pool: poolInfo, + }; + } + + // TODO consider breaking out rendering search doc into its own route so + // that we can fully understand all the linked fields that are used in all + // the html formats and generate a search doc that is well populated. Right + // now we only consider linked fields used in the isolated template. + let metaMaybeError = await withTimeout( + page, + () => renderMeta(page, captureOptions), + opts?.timeoutMs, + ); + // TODO also consider introducing a mechanism in the API to track and reset + // field usage for an instance recursively so that the depth that an + // instance is loaded from a different rendering context in the same realm + // doesn't elide fields that this rendering context cares about. in that + // manner we can get a complete picture of how to build the search doc's linked + // fields for each rendering context. + let meta: PrerenderMeta; + if (isRenderError(metaMaybeError)) { + markTimeout(metaMaybeError as RenderError); + if ( + await this.#maybeEvict( + realm, + 'render.meta', + metaMaybeError as RenderError, + ) + ) { + poolInfo.evicted = true; + shortCircuit = true; + } + error = error ?? (metaMaybeError as RenderError); + markTimeout(error); + meta = { + serialized: null, + searchDoc: null, + displayNames: null, + deps: null, + types: null, + }; + } else { + meta = metaMaybeError; + } + let atomHTML: string | null = null, + iconHTML: string | null = null, + embeddedHTML: Record | null = null, + fittedHTML: Record | null = null; + if (!shortCircuit && meta.types) { + // Render sequentially and short-circuit on unusable page/timeout + const steps: Array<{ + name: string; + cb: () => Promise | RenderError>; + assign: (value: string | Record) => void; + }> = [ + { + name: 'fitted render', + cb: () => + renderAncestors(page, 'fitted', meta.types!, captureOptions), + assign: (v: string | Record) => { + fittedHTML = v as Record; }, - { - name: 'embedded render', - cb: () => - renderAncestors(page, 'embedded', meta.types!, captureOptions), - assign: (v: string | Record) => { - embeddedHTML = v as Record; - }, + }, + { + name: 'embedded render', + cb: () => + renderAncestors(page, 'embedded', meta.types!, captureOptions), + assign: (v: string | Record) => { + embeddedHTML = v as Record; }, - { - name: 'atom render', - cb: () => renderHTML(page, 'atom', 0, captureOptions), - assign: (v: string | Record) => { - atomHTML = v as string; - }, + }, + { + name: 'atom render', + cb: () => renderHTML(page, 'atom', 0, captureOptions), + assign: (v: string | Record) => { + atomHTML = v as string; }, - { - name: 'icon render', - cb: () => renderIcon(page, captureOptions), - assign: (v: string | Record) => { - iconHTML = v as string; - }, + }, + { + name: 'icon render', + cb: () => renderIcon(page, captureOptions), + assign: (v: string | Record) => { + iconHTML = v as string; }, - ]; + }, + ]; - for (let step of steps) { - if (shortCircuit) break; - let res = await this.#step(realm, step.name, () => - withTimeout(page, step.cb, opts?.timeoutMs), - ); - if (res.ok) { - step.assign(res.value); - } else { - error = error ?? res.error; - markTimeout(res.error); - if (res.evicted) { - poolInfo.evicted = true; - shortCircuit = true; - break; - } + for (let step of steps) { + if (shortCircuit) break; + let res = await this.#step(realm, step.name, () => + withTimeout(page, step.cb, opts?.timeoutMs), + ); + if (res.ok) { + step.assign(res.value); + } else { + error = error ?? res.error; + markTimeout(res.error); + if (res.evicted) { + poolInfo.evicted = true; + shortCircuit = true; + break; } } } + } - let response: RenderResponse = { - ...(meta as PrerenderMeta), - ...(error ? { error } : {}), - iconHTML, - isolatedHTML, - atomHTML, - embeddedHTML, - fittedHTML, - }; - return { - response, - timings: { launchMs, renderMs: Date.now() - renderStart }, - pool: poolInfo, - }; - } finally { - deferred.fulfill(); + let response: RenderResponse = { + ...(meta as PrerenderMeta), + ...(error ? { error } : {}), + iconHTML, + isolatedHTML, + atomHTML, + embeddedHTML, + fittedHTML, + }; + return { + response, + timings: { launchMs, renderMs: Date.now() - renderStart }, + pool: poolInfo, + }; + } + + #shouldRetryWithClearCache( + response: RenderResponse, + ): readonly string[] | undefined { + let renderError = response.error?.error; + if (!renderError) { + return undefined; + } + let parts = [renderError.message, renderError.stack].filter( + (part): part is string => typeof part === 'string' && part.length > 0, + ); + if (parts.length === 0) { + return undefined; + } + let haystack = parts.join('\n'); + for (let signature of CLEAR_CACHE_RETRY_SIGNATURES) { + if (signature.every((fragment) => haystack.includes(fragment))) { + return signature; + } } + return undefined; } async #getBrowser(): Promise { diff --git a/packages/realm-server/prerender/utils.ts b/packages/realm-server/prerender/utils.ts index 9cb4a75e2d4..80da5fe3104 100644 --- a/packages/realm-server/prerender/utils.ts +++ b/packages/realm-server/prerender/utils.ts @@ -1,4 +1,5 @@ import { + cleanCapturedHTML, delay, logger, type PrerenderMeta, @@ -56,7 +57,7 @@ export async function renderHTML( if (result.status === 'error' || result.status === 'unusable') { return renderCaptureToError(page, result, 'render.html'); } - return result.value; + return cleanCapturedHTML(result.value); } export async function renderIcon( @@ -68,7 +69,7 @@ export async function renderIcon( if (result.status === 'error' || result.status === 'unusable') { return renderCaptureToError(page, result, 'render.icon'); } - return result.value; + return cleanCapturedHTML(result.value); } export async function renderMeta( diff --git a/packages/realm-server/scripts/start-all-except-experiments.sh b/packages/realm-server/scripts/start-all-except-experiments.sh index 08bfd7420c0..1ebef24d59d 100755 --- a/packages/realm-server/scripts/start-all-except-experiments.sh +++ b/packages/realm-server/scripts/start-all-except-experiments.sh @@ -1,6 +1,6 @@ #! /bin/sh -SKIP_EXPERIMENTS=true NODE_NO_WARNINGS=1 start-server-and-test \ +WAIT_ON_TIMEOUT=1200000 SKIP_EXPERIMENTS=true NODE_NO_WARNINGS=1 start-server-and-test \ 'run-p start:pg start:prerender-dev start:matrix start:smtp start:worker-development start:development' \ 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001' \ 'run-p start:worker-test start:test-realms' \ diff --git a/packages/realm-server/scripts/start-all.sh b/packages/realm-server/scripts/start-all.sh index 0cb3c08c7d4..4c4520b500a 100755 --- a/packages/realm-server/scripts/start-all.sh +++ b/packages/realm-server/scripts/start-all.sh @@ -1,21 +1,25 @@ #! /bin/sh BASE_REALM="http-get://localhost:4201/base/" +CATALOG_REALM="http-get://localhost:4201/catalog/" +SKILLS_REALM="http-get://localhost:4201/skills/" EXPERIMENTS_REALM="http-get://localhost:4201/experiments/" NODE_TEST_REALM="http-get://localhost:4202/node-test/" READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" BASE_REALM_READY="$BASE_REALM$READY_PATH" +CATALOG_REALM_READY="$CATALOG_REALM$READY_PATH" +SKILLS_REALM_READY="$SKILLS_REALM$READY_PATH" EXPERIMENTS_REALM_READY="$EXPERIMENTS_REALM$READY_PATH" NODE_TEST_REALM_READY="$NODE_TEST_REALM$READY_PATH" SYNAPSE_URL="http://localhost:8008" SMTP_4_DEV_URL="http://localhost:5001" -NODE_NO_WARNINGS=1 start-server-and-test \ +WAIT_ON_TIMEOUT=1200000 NODE_NO_WARNINGS=1 start-server-and-test \ 'run-p start:pg start:matrix start:smtp start:prerender-dev start:worker-development start:development' \ - "$BASE_REALM_READY|$EXPERIMENTS_REALM_READY|$SYNAPSE_URL|$SMTP_4_DEV_URL" \ + "$BASE_REALM_READY|$CATALOG_REALM_READY|$SKILLS_REALM_READY|$EXPERIMENTS_REALM_READY|$SYNAPSE_URL|$SMTP_4_DEV_URL" \ 'run-p start:worker-test start:test-realms' \ "$NODE_TEST_REALM_READY" \ 'wait' diff --git a/packages/realm-server/scripts/start-services-for-matrix-tests.sh b/packages/realm-server/scripts/start-services-for-matrix-tests.sh index b6b45c03552..9266caebe4a 100755 --- a/packages/realm-server/scripts/start-services-for-matrix-tests.sh +++ b/packages/realm-server/scripts/start-services-for-matrix-tests.sh @@ -1,5 +1,5 @@ #! /bin/sh -NODE_NO_WARNINGS=1 start-server-and-test \ +WAIT_ON_TIMEOUT=600000 NODE_NO_WARNINGS=1 start-server-and-test \ 'run-p start:pg start:prerender-dev start:worker-base start:base' \ 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \ 'run-p start:worker-test start:test-realms' \ diff --git a/packages/realm-server/scripts/start-without-matrix.sh b/packages/realm-server/scripts/start-without-matrix.sh index c1a9b974268..23dd256e128 100755 --- a/packages/realm-server/scripts/start-without-matrix.sh +++ b/packages/realm-server/scripts/start-without-matrix.sh @@ -15,7 +15,7 @@ TEST_REALM_READY="$TEST_REALM$READY_PATH" SYNAPSE_URL="http://localhost:8008" SMTP_4_DEV_URL="http://localhost:5001" -NODE_NO_WARNINGS=1 start-server-and-test \ +WAIT_ON_TIMEOUT=1200000 NODE_NO_WARNINGS=1 start-server-and-test \ 'run-p start:pg start:prerender-dev start:worker-development start:development' \ "$BASE_REALM_READY|$EXPERIMENTS_REALM_READY|$SYNAPSE_URL|$SMTP_4_DEV_URL" \ 'run-p start:worker-test start:test-realms' \ diff --git a/packages/realm-server/tests/prerendering-test.ts b/packages/realm-server/tests/prerendering-test.ts index 7a0948c37a7..797aabc0aa3 100644 --- a/packages/realm-server/tests/prerendering-test.ts +++ b/packages/realm-server/tests/prerendering-test.ts @@ -13,6 +13,7 @@ import { setupPermissionedRealms, matrixURL, realmSecretSeed, + cleanWhiteSpace, } from './helpers'; import '@cardstack/runtime-common/helpers/code-equality-assertion'; import { baseCardRef } from '@cardstack/runtime-common'; @@ -483,7 +484,7 @@ module(basename(__filename), function () { test('embedded HTML', function (assert) { assert.ok( /Maple\s+says\s+Meow/.test( - result.embeddedHTML![`${realmURL2}cat/Cat`], + cleanWhiteSpace(result.embeddedHTML![`${realmURL2}cat/Cat`]), ), `failed to match embedded html:${JSON.stringify(result.embeddedHTML)}`, ); diff --git a/packages/runtime-common/html-utils.ts b/packages/runtime-common/html-utils.ts new file mode 100644 index 00000000000..f1dfcd78fa4 --- /dev/null +++ b/packages/runtime-common/html-utils.ts @@ -0,0 +1,10 @@ +export function cleanCapturedHTML(html: string): string { + if (!html) { + return html; + } + const emberIdAttr = /\s+id=(?:"ember\d+"|'ember\d+'|ember\d+)(?=[\s>])/g; + const emptyDataAttr = /\s+(data-[A-Za-z0-9:_-]+)=(?:""|''|(?=[\s>]))/g; + let cleaned = html.replace(emberIdAttr, ''); + cleaned = cleaned.replace(emptyDataAttr, ' $1'); + return cleaned; +} diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index d683f20cc22..53e174a991d 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -137,6 +137,7 @@ export * from './stream'; export * from './realm'; export * from './fetcher'; export * from './scoped-css'; +export * from './html-utils'; export * from './utils'; export * from './authorization-middleware'; export * from './resource-types'; diff --git a/packages/runtime-common/realm-index-updater.ts b/packages/runtime-common/realm-index-updater.ts index 7b93f41bcd1..47f82b62a19 100644 --- a/packages/runtime-common/realm-index-updater.ts +++ b/packages/runtime-common/realm-index-updater.ts @@ -19,6 +19,9 @@ import { Realm } from './realm'; import { RealmPaths } from './paths'; import ignore, { type Ignore } from 'ignore'; +const FROM_SCRATCH_JOB_TIMEOUT_SEC = 10 * 60; +const INCREMENTAL_JOB_TIMEOUT_SEC = 3 * 60; + export class RealmIndexUpdater { #realm: Realm; #log = logger('realm-index-updater'); @@ -93,7 +96,7 @@ export class RealmIndexUpdater { let job = await this.#queue.publish({ jobType: `from-scratch-index`, concurrencyGroup: `indexing:${this.#realm.url}`, - timeout: 3 * 60, + timeout: FROM_SCRATCH_JOB_TIMEOUT_SEC, priority: systemInitiatedPriority, args, }); @@ -130,7 +133,7 @@ export class RealmIndexUpdater { let job = await this.#queue.publish({ jobType: `incremental-index`, concurrencyGroup: `indexing:${this.#realm.url}`, - timeout: 60, + timeout: INCREMENTAL_JOB_TIMEOUT_SEC, priority: userInitiatedPriority, args, });