Feedback forms: tab capture for the marked screenshot and the recording (Gleap #228450)

src/Gleap.js

@@ -78,6 +78,8 @@ class Gleap {
   // Global data
   globalData = {
     screenRecordingData: null,
+    // The feedback form's marked screenshot when it was taken from the tab (GleapMarkerManager).
+    screenshotImage: null,
     webReplay: null,
     snapshotPosition: {
       x: 0,

src/GleapCaptureManager.js

@@ -22,10 +22,10 @@ import {
   CAPTURE_MAX_RECORDING_SEC,
   CAPTURE_PLATFORM,
   CAPTURE_SDK_TYPE,
+  canCaptureTab,
   getCaptureConfig,
   getRecordingMethod,
   getSdkVersion,
-  hasDisplayMedia,
   hasUserMedia,
   isCaptureSupported,
   isDesktopChromium,
@@ -83,14 +83,6 @@ const HIDDEN_GLEAP_UI_SELECTOR = [
   'gleap-checklist',
 ].join(', ');
 
-const isTopWindow = () => {
-  try {
-    return window.top === window.self;
-  } catch (exp) {
-    return false;
-  }
-};
-
 const readStorage = (key) => {
   try {
     return window.sessionStorage.getItem(key);
@@ -680,10 +672,8 @@ export default class GleapCaptureManager {
     if (session.kind !== 'screenshot' || session.phase !== 'bar') {
       return;
     }
-    const config = getCaptureConfig();
-    // Called inside the Capture click: the tab capture needs that user activation. Not from inside a
-    // frame: the tab shows the whole page, the masks only know this frame.
-    const useTab = config.webScreenshotMethod !== 'dom-only' && isDesktopChromium() && hasDisplayMedia() && isTopWindow();
+    // Called inside the Capture click: the tab capture needs that user activation.
+    const useTab = canCaptureTab();
     const streamPromise = useTab ? requestTabStream() : null;
 
     session.phase = 'capturing';
@@ -999,9 +989,7 @@ export default class GleapCaptureManager {
     session.recording = null;
     session.result = result;
     // When it was paused: the logs leave those stretches out (Pause is for logging in or paying).
-    session.pauses = session.timeline
-      ? session.timeline.pauses.map((pause) => [pause[0], pause[1] || result.endedAt])
-      : [];
+    session.pauses = session.timeline ? session.timeline.pauses.map((pause) => [pause[0], pause[1] || result.endedAt]) : [];
     session.timelineEntries = session.timeline ? session.timeline.stop(result.endedAt) : [];
     const interactions = session.timeline ? session.timeline.interactions : 0;
     session.timeline = null;

src/GleapCaptureScreenshot.js

@@ -1554,14 +1554,9 @@ export const renderDom = ({ privacyOptions, maskSelectors, isActive }) =>
       .catch((error) => finish(error));
   });
 
-/**
- * Takes the screenshot: the shared tab when `streamPromise` (from requestTabStream) delivers one,
- * the DOM render otherwise (declined, cancelled, another surface or any error) — without asking again.
- * @returns {Promise<{dataUrl: string, width: number, height: number, method: 'tab'|'dom'}>}
- */
-export const captureScreenshot = ({ streamPromise, setUiHidden, privacyOptions, maskSelectors, isActive }) => {
-  const active = () => !isActive || isActive();
-  const fromTab = streamPromise
+// One frame of the shared tab, or null when there is no stream, it was declined or it failed.
+const captureTab = ({ streamPromise, setUiHidden, privacyOptions, maskSelectors, active }) =>
+  streamPromise
     ? streamPromise.then(
         (stream) => {
           if (!active()) {
@@ -1573,15 +1568,51 @@ export const captureScreenshot = ({ streamPromise, setUiHidden, privacyOptions,
           );
         },
         () => {
-          // Declined or failed: the bar (hidden for the share dialog) shows its busy state again
-          // while the page is rendered instead.
+          // Declined or failed: the UI (hidden for the share dialog) shows again.
           setUiHidden(false);
           return null;
         }
       )
     : Promise.resolve(null);
 
-  return fromTab
+const encodeAndRelease = (capture) => {
+  try {
+    const encoded = encodeCapture(capture);
+    encoded.method = capture.method;
+    return encoded;
+  } finally {
+    releaseCanvas(capture.canvas);
+  }
+};
+
+/**
+ * Takes the screenshot from the shared tab only (`streamPromise` from requestTabStream). Resolves
+ * with null when the tab was declined, cancelled, another surface or failed: the caller has its own
+ * fallback (the feedback form renders the DOM on the server).
+ * @returns {Promise<{dataUrl: string, width: number, height: number, method: 'tab'}|null>}
+ */
+export const captureTabScreenshot = ({ streamPromise, setUiHidden, privacyOptions, maskSelectors, isActive }) => {
+  const active = () => !isActive || isActive();
+  return captureTab({ streamPromise, setUiHidden, privacyOptions, maskSelectors, active }).then((capture) => {
+    if (!capture) {
+      return null;
+    }
+    if (!active()) {
+      releaseCanvas(capture.canvas);
+      return null;
+    }
+    return encodeAndRelease(capture);
+  });
+};
+
+/**
+ * Takes the screenshot: the shared tab when `streamPromise` (from requestTabStream) delivers one,
+ * the DOM render otherwise (declined, cancelled, another surface or any error) — without asking again.
+ * @returns {Promise<{dataUrl: string, width: number, height: number, method: 'tab'|'dom'}>}
+ */
+export const captureScreenshot = ({ streamPromise, setUiHidden, privacyOptions, maskSelectors, isActive }) => {
+  const active = () => !isActive || isActive();
+  return captureTab({ streamPromise, setUiHidden, privacyOptions, maskSelectors, active })
     .then((capture) => {
       if (!active()) {
         if (capture) {
@@ -1591,13 +1622,5 @@ export const captureScreenshot = ({ streamPromise, setUiHidden, privacyOptions,
       }
       return capture || renderDom({ privacyOptions, maskSelectors, isActive: active });
     })
-    .then((capture) => {
-      try {
-        const encoded = encodeCapture(capture);
-        encoded.method = capture.method;
-        return encoded;
-      } finally {
-        releaseCanvas(capture.canvas);
-      }
-    });
+    .then(encodeAndRelease);
 };

src/GleapCaptureSettings.js

@@ -124,6 +124,22 @@ export const isDesktopChromium = () => {
   }
 };
 
+const isTopWindow = () => {
+  try {
+    return window.top === window.self;
+  } catch (exp) {
+    return false;
+  }
+};
+
+/**
+ * Whether a screenshot can be one frame of the current tab (desktop Chromium, unless the project
+ * renders the DOM only). Not from inside a frame: the tab shows the whole page, the masks only know
+ * this frame. The request itself needs a user activation (requestTabStream inside the click).
+ */
+export const canCaptureTab = () =>
+  getCaptureConfig().webScreenshotMethod !== 'dom-only' && isDesktopChromium() && hasDisplayMedia() && isTopWindow();
+
 /**
  * 'display' (getDisplayMedia + MediaRecorder, desktop browsers) or 'rrweb' (a page recording,
  * phones and tablets).

src/GleapEventManager.js

@@ -20,6 +20,7 @@ export default class GleapEventManager {
       const gleapInstance = Gleap.getInstance();
       gleapInstance.setGlobalDataItem('webReplay', null);
       gleapInstance.setGlobalDataItem('screenRecordingData', null);
+      gleapInstance.setGlobalDataItem('screenshotImage', null);
       gleapInstance.takeCurrentReplay();
     }
 

src/GleapFeedback.js

@@ -17,6 +17,17 @@ import Gleap, {
 // spinner that never ends. Generous on purpose — a 20k-element page captures in ~1.5s.
 const SCREENSHOT_CAPTURE_TIMEOUT_MS = 15000;
 
+const dataUrlToFile = (dataUrl, fileName) => {
+  const [header, base64] = dataUrl.split(',');
+  const type = (/^data:([^;]+)/.exec(header) || [])[1] || 'image/jpeg';
+  const binary = atob(base64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return new File([bytes], fileName, { type });
+};
+
 // Resolves early if the wrapped promise overruns. Rejection is passed through untouched, so a
 // genuinely failed capture still surfaces as a failed send exactly as before.
 const resolveAfterTimeout = (promise, timeoutMs, label) =>
@@ -71,9 +82,12 @@ export default class GleapFeedback {
   isSilent = false;
   outboundId = undefined;
   screenshotData = undefined;
+  // The marked screenshot as an uploaded image (see forgetSentCaptures); screenshotData otherwise.
+  screenshotUrl = undefined;
+  screenshotImage = undefined;
   webReplay = undefined;
   screenRecordingUrl = undefined;
-  // The form's recording this report uploads (see forgetSentScreenRecording).
+  // The form's recording this report uploads (see forgetSentCaptures).
   screenRecordingData = undefined;
   spamToken = undefined;
 
@@ -109,7 +123,7 @@ export default class GleapFeedback {
     var screenRecordingData = this.isSilent ? null : gleapInstance.getGlobalDataItem('screenRecordingData');
     if (screenRecordingData != null) {
       this.screenRecordingData = screenRecordingData;
-      var recordingUrlPromise = GleapScreenRecorder.uploadScreenRecording(screenRecordingData).then((recordingUrl) => {
+      var recordingUrlPromise = GleapScreenRecorder.uploadFile(screenRecordingData).then((recordingUrl) => {
         if (recordingUrl) {
           this.screenRecordingUrl = recordingUrl;
         }
@@ -119,21 +133,35 @@ export default class GleapFeedback {
 
     // Prepare screenshot
     if (!(this.excludeData && this.excludeData.screenshot)) {
-      // The replay options go along so form fields are masked the same way as in replays.
-      var screenshotDataPromise = startScreenCapture(
-        gleapInstance.isLiveMode(),
-        GleapReplayRecorder.getInstance().customOptions
-      ).then((screenshotData) => {
-        if (screenshotData) {
-          const snapshotPosition = gleapInstance.getGlobalDataItem('snapshotPosition');
-          screenshotData['x'] = snapshotPosition.x;
-          screenshotData['y'] = snapshotPosition.y;
-          this.screenshotData = screenshotData;
-        }
-      });
-      dataPromises.push(
-        resolveAfterTimeout(screenshotDataPromise, SCREENSHOT_CAPTURE_TIMEOUT_MS, 'screenshot capture')
-      );
+      // The DOM, rendered on the server. The replay options go along so form fields are masked the
+      // same way as in replays.
+      const captureDom = () =>
+        startScreenCapture(gleapInstance.isLiveMode(), GleapReplayRecorder.getInstance().customOptions).then(
+          (screenshotData) => {
+            if (screenshotData) {
+              const snapshotPosition = gleapInstance.getGlobalDataItem('snapshotPosition');
+              screenshotData['x'] = snapshotPosition.x;
+              screenshotData['y'] = snapshotPosition.y;
+              this.screenshotData = screenshotData;
+            }
+          }
+        );
+      // The screenshot the user marked, when the form took it from the tab (GleapMarkerManager): it
+      // goes out as an image. Without one (no tab capture in this browser, declined, a report sent
+      // without marking, a silent report) or when its upload fails, the DOM is rendered instead.
+      const screenshotImage = this.isSilent ? null : gleapInstance.getGlobalDataItem('screenshotImage');
+      const screenshotPromise = screenshotImage
+        ? GleapScreenRecorder.uploadFile(dataUrlToFile(screenshotImage.dataUrl, 'screenshot.jpg'))
+            .then((screenshotUrl) => {
+              if (!screenshotUrl) {
+                throw new Error('no-screenshot-url');
+              }
+              this.screenshotImage = screenshotImage;
+              this.screenshotUrl = screenshotUrl;
+            })
+            .catch(captureDom)
+        : captureDom();
+      dataPromises.push(resolveAfterTimeout(screenshotPromise, SCREENSHOT_CAPTURE_TIMEOUT_MS, 'screenshot capture'));
     }
 
     return Promise.all(dataPromises);
@@ -156,6 +184,7 @@ export default class GleapFeedback {
       isSilent: this.isSilent,
       outbound: this.outboundId,
       screenshotData: this.screenshotData,
+      screenshotUrl: this.screenshotUrl,
       webReplay: this.webReplay,
       screenRecordingUrl: this.screenRecordingUrl,
       spamToken: this.spamToken,
@@ -177,6 +206,7 @@ export default class GleapFeedback {
 
           if (keyToExclude === 'screenshot') {
             delete feedbackData.screenshotData;
+            delete feedbackData.screenshotUrl;
           }
 
           if (keyToExclude === 'replays') {
@@ -203,13 +233,16 @@ export default class GleapFeedback {
     });
   }
 
-  // The recording went out with this report: a later report must not upload it again.
-  forgetSentScreenRecording() {
+  // The recording and the marked screenshot went out with this report: a later report must not
+  // upload them again.
+  forgetSentCaptures() {
     try {
       const gleapInstance = Gleap.getInstance();
-      if (this.screenRecordingData && gleapInstance.getGlobalDataItem('screenRecordingData') === this.screenRecordingData) {
-        gleapInstance.setGlobalDataItem('screenRecordingData', null);
-      }
+      ['screenRecordingData', 'screenshotImage'].forEach((key) => {
+        if (this[key] && gleapInstance.getGlobalDataItem(key) === this[key]) {
+          gleapInstance.setGlobalDataItem(key, null);
+        }
+      });
     } catch (exp) {}
   }
 
@@ -231,7 +264,7 @@ export default class GleapFeedback {
             http.onreadystatechange = function (e) {
               if (http.readyState === 4) {
                 if (http.status === 200 || http.status === 201) {
-                  self.forgetSentScreenRecording();
+                  self.forgetSentCaptures();
                   try {
                     const feedback = JSON.parse(http.responseText);
                     resolve(feedback);