Add ability to prime URL metrics

.github/workflows/js-lint.yml

@@ -45,7 +45,9 @@ jobs:
           node-version-file: '.nvmrc'
           cache: npm
       - name: npm install
-        run: npm ci
+        run: |
+          npm ci
+          npm ci --prefix plugins/optimization-detective/priming-cli
       - name: JS Lint
         run: npm run lint-js
       - name: TypeScript compile

plugins/optimization-detective/class-od-url-metric.php

@@ -43,7 +43,8 @@
  *                                url: non-empty-string,
  *                                timestamp: float,
  *                                viewport: ViewportRect,
- *                                elements: ElementData[]
+ *                                elements: ElementData[],
+ *                                source?: non-empty-string,
  *                            }
  * @phpstan-type JSONSchema   array{
  *                                type: string|string[],
@@ -310,6 +311,16 @@ public static function get_json_schema(): array {
 						'additionalProperties' => true,
 					),
 				),
+				'source'    => array(
+					'description' => __( 'The source of the URL Metric.', 'optimization-detective' ),
+					'type'        => 'string',
+					'required'    => false,
+					'enum'        => array(
+						'visitor',
+						'user',
+						'synthetic',
+					),
+				),
 			),
 			// Additional root properties may be added to the schema via the od_url_metric_schema_root_additional_properties filter.
 			// Therefore, additionalProperties is set to true so that additional properties defined in the extended schema may persist
@@ -526,6 +537,17 @@ function ( array $element ): OD_Element {
 		return $this->elements;
 	}
 
+	/**
+	 * Gets the source of the URL Metric.
+	 *
+	 * @since n.e.x.t
+	 *
+	 * @return non-empty-string|null Source.
+	 */
+	public function get_source(): ?string {
+		return $this->data['source'] ?? null;
+	}
+
 	/**
 	 * Specifies data which should be serialized to JSON.
 	 *

plugins/optimization-detective/detect.js

@@ -65,6 +65,14 @@ const storageLockTimeSessionKey = 'odStorageLockTime';
  */
 const compressionDebounceWaitDuration = 1000;
 
+/**
+ * Verification token for skipping the storage lock check while priming URL Metrics.
+ *
+ * @see {detect}
+ * @type {?string}
+ */
+let odPrimeUrlMetricsVerificationToken = null;
+
 /**
  * Checks whether storage is locked.
  *
@@ -73,6 +81,10 @@ const compressionDebounceWaitDuration = 1000;
  * @return {boolean} Whether storage is locked.
  */
 function isStorageLocked( currentTime, storageLockTTL ) {
+	if ( odPrimeUrlMetricsVerificationToken ) {
+		return false;
+	}
+
 	if ( storageLockTTL === 0 ) {
 		return false;
 	}
@@ -503,6 +515,116 @@ function debounceCompressUrlMetric() {
 	}, compressionDebounceWaitDuration );
 }
 
+/**
+ * Forces immediate compression of the URL Metric, bypassing debounce and idle callbacks.
+ *
+ * @return {Promise<void>} A promise that resolves when the compression is complete.
+ */
+async function forceCompressUrlMetric() {
+	if ( ! compressionEnabled ) {
+		return null;
+	}
+	if ( null !== recompressionTimeout ) {
+		clearTimeout( recompressionTimeout );
+		recompressionTimeout = null;
+	}
+	if (
+		null !== idleCallbackHandle &&
+		typeof cancelIdleCallback === 'function'
+	) {
+		cancelIdleCallback( idleCallbackHandle );
+		idleCallbackHandle = null;
+	}
+
+	try {
+		compressedPayload = await compress( JSON.stringify( urlMetric ) );
+	} catch ( err ) {
+		const { error } = createLogger( false, consoleLogPrefix );
+		error(
+			'Failed to compress URL Metric falling back to sending uncompressed data:',
+			err
+		);
+		compressionEnabled = false;
+	}
+}
+
+/**
+ * Notifies about the URL Metric request status.
+ *
+ * @param {Object}  status                  - The status details.
+ * @param {boolean} status.success          - Indicates if the request succeeded.
+ * @param {string}  [status.error]          - An error message if the request failed.
+ * @param {Object}  [options]               - Options for where to dispatch the message.
+ * @param {boolean} [options.toParent=true] - Whether to send the message to the parent window.
+ * @param {boolean} [options.toLocal=true]  - Whether to dispatch a custom event locally.
+ */
+function notifyStatus( status, options = { toParent: true, toLocal: true } ) {
+	const message = {
+		type: 'OD_PRIME_URL_METRICS_REQUEST_STATUS',
+		success: status.success,
+		...( status.error && { error: status.error } ),
+	};
+
+	// This will be used when URL metrics are primed using a IFRAME.
+	if ( options.toParent && window.parent && window.parent !== window ) {
+		window.parent.postMessage( message, '*' );
+	}
+
+	// This will be used when URL metrics are primed using Puppeteer script.
+	if ( options.toLocal ) {
+		document.dispatchEvent(
+			new CustomEvent( message.type, {
+				detail: { ...status },
+			} )
+		);
+	}
+}
+
+/**
+ * Scrolls to the bottom of the page.
+ *
+ * @return {Promise<void>} A promise that resolves when the scroll is complete.
+ */
+async function scrollToBottomOfPage() {
+	return new Promise( ( resolve ) => {
+		const viewportHeight = window.innerHeight;
+		const maxScrollAttempts = 20;
+		const maxHeightChangeAmount = viewportHeight * 0.5;
+		const maxHeightChangeAttempts = 3;
+
+		let scrollAttempts = 0;
+		let heightChangeAttempts = 0;
+		let lastScrollPosition = 0;
+		let lastHeight = document.documentElement.scrollHeight;
+
+		function scroll() {
+			const newHeight = document.documentElement.scrollHeight;
+			if (
+				window.innerHeight + window.scrollY >= newHeight - 10 ||
+				scrollAttempts >= maxScrollAttempts ||
+				heightChangeAttempts >= maxHeightChangeAttempts
+			) {
+				resolve();
+				return;
+			}
+
+			lastScrollPosition += viewportHeight;
+			window.scrollTo( { top: lastScrollPosition, behavior: 'smooth' } );
+
+			const heightChange = newHeight - lastHeight;
+			if ( heightChange >= maxHeightChangeAmount ) {
+				lastHeight = newHeight;
+				heightChangeAttempts++;
+			}
+
+			scrollAttempts++;
+			setTimeout( scroll, 300 );
+		}
+
+		scroll();
+	} );
+}
+
 /**
  * @typedef {{timestamp: number, creationDate: Date}} UrlMetricDebugData
  * @typedef {{groups: Array<{url_metrics: Array<UrlMetricDebugData>}>}} CollectionDebugData
@@ -595,6 +717,15 @@ export default async function detect( {
 		return;
 	}
 
+	// Retrieve verification token from the URL hash for priming URL Metrics.
+	// Presence of the token indicates that the URL Metric is being primed
+	// through the Puppeteer script or WordPress admin dashboard.
+	if ( '' !== window.location.hash ) {
+		odPrimeUrlMetricsVerificationToken = new URLSearchParams(
+			window.location.hash.slice( 1 )
+		).get( 'odPrimeUrlMetricsVerificationToken' );
+	}
+
 	// Abort if the client already submitted a URL Metric for this URL and viewport group.
 	const alreadySubmittedSessionStorageKey =
 		await getAlreadySubmittedSessionStorageKey(
@@ -604,6 +735,7 @@ export default async function detect( {
 			logger
 		);
 	if (
+		! odPrimeUrlMetricsVerificationToken &&
 		null !== alreadySubmittedSessionStorageKey &&
 		alreadySubmittedSessionStorageKey in sessionStorage
 	) {
@@ -787,8 +919,13 @@ export default async function detect( {
 			height: win.innerHeight,
 		},
 		elements: [],
+		source: restApiNonce ? 'user' : 'visitor',
 	};
 
+	if ( odPrimeUrlMetricsVerificationToken ) {
+		urlMetric.source = 'synthetic';
+	}
+
 	const lcpMetric = lcpMetricCandidates.at( -1 );
 
 	// Populate the elements in the URL Metric.
@@ -921,19 +1058,25 @@ export default async function detect( {
 	debounceCompressUrlMetric();
 
 	// Wait for the page to be hidden.
-	await new Promise( ( resolve ) => {
-		win.addEventListener( 'pagehide', resolve, { once: true } );
-		win.addEventListener( 'pageswap', resolve, { once: true } );
-		doc.addEventListener(
-			'visibilitychange',
-			() => {
-				if ( document.visibilityState === 'hidden' ) {
-					// TODO: This will fire even when switching tabs.
-					resolve();
-				}
-			},
-			{ once: true }
-		);
+	await new Promise( async ( resolve ) => {
+		if ( ! odPrimeUrlMetricsVerificationToken ) {
+			win.addEventListener( 'pagehide', resolve, { once: true } );
+			win.addEventListener( 'pageswap', resolve, { once: true } );
+			doc.addEventListener(
+				'visibilitychange',
+				() => {
+					if ( document.visibilityState === 'hidden' ) {
+						// TODO: This will fire even when switching tabs.
+						resolve();
+					}
+				},
+				{ once: true }
+			);
+		} else {
+			await scrollToBottomOfPage();
+			await forceCompressUrlMetric();
+			resolve();
+		}
 	} );
 
 	// Only proceed with submitting the URL Metric if viewport stayed the same size. Changing the viewport size (e.g. due
@@ -1092,6 +1235,12 @@ export default async function detect( {
 		);
 	}
 	url.searchParams.set( 'hmac', urlMetricHMAC );
+	if ( odPrimeUrlMetricsVerificationToken ) {
+		url.searchParams.set(
+			'prime_url_metrics_verification_token',
+			odPrimeUrlMetricsVerificationToken
+		);
+	}
 
 	const headers = {
 		'Content-Type': 'application/json',
@@ -1104,7 +1253,22 @@ export default async function detect( {
 		method: 'POST',
 		body: payloadBlob,
 		headers,
-		keepalive: true, // This makes fetch() behave the same as navigator.sendBeacon().
+		keepalive: odPrimeUrlMetricsVerificationToken ? false : true, // Setting keepalive to true makes fetch() behave the same as navigator.sendBeacon().
 	} );
-	await fetch( request );
+
+	if ( ! odPrimeUrlMetricsVerificationToken ) {
+		await fetch( request );
+	} else {
+		try {
+			const response = await fetch( request );
+			if ( ! response.ok ) {
+				throw new Error(
+					`Failed to send URL Metric. Status: ${ response.status }`
+				);
+			}
+			notifyStatus( { success: true } );
+		} catch ( err ) {
+			notifyStatus( { success: false, error: err.message } );
+		}
+	}
 }

plugins/optimization-detective/detection.php

@@ -236,6 +236,31 @@ function_exists( 'gzdecode' ) &&
 	return $result;
 }
 
+/**
+ * Registers the REST API endpoint for priming URL Metrics.
+ *
+ * @since n.e.x.t
+ * @access private
+ */
+function od_register_rest_url_metric_priming_endpoint(): void {
+	$endpoint_controller = new OD_REST_URL_Metrics_Priming_Endpoint();
+	register_rest_route(
+		OD_REST_URL_Metrics_Store_Endpoint::ROUTE_NAMESPACE,
+		$endpoint_controller::PRIME_URLS_ROUTE,
+		$endpoint_controller->get_registration_args_prime_urls()
+	);
+	register_rest_route(
+		OD_REST_URL_Metrics_Store_Endpoint::ROUTE_NAMESPACE,
+		$endpoint_controller::PRIME_URLS_BREAKPOINTS_ROUTE,
+		$endpoint_controller->get_registration_args_prime_urls_breakpoints()
+	);
+	register_rest_route(
+		OD_REST_URL_Metrics_Store_Endpoint::ROUTE_NAMESPACE,
+		$endpoint_controller::PRIME_URLS_VERIFICATION_TOKEN_ROUTE,
+		$endpoint_controller->get_registration_args_prime_urls_verification_token()
+	);
+}
+
 /**
  * Triggers post update actions for page caches to invalidate their caches related to the supplied cache purge post ID.
  *