diff --git a/.github/workflows/enterprise.yml b/.github/workflows/enterprise.yml
index e304ee5a459db..e4c9ae0dd39b1 100644
--- a/.github/workflows/enterprise.yml
+++ b/.github/workflows/enterprise.yml
@@ -250,6 +250,16 @@ jobs:
. venv_tests/bin/activate
MOZ_LOG=console:5 python3 test_felt_browser_signout.py ${{ steps.unpack.outputs.runtime_path }} $PWD/geckodriver $PWD/profiles/
+ - name: "test browser crash handled"
+ run: |
+ . venv_tests/bin/activate
+ MOZ_LOG=console:5 python3 test_felt_browser_crash_abort_restart.py ${{ steps.unpack.outputs.runtime_path }} $PWD/geckodriver $PWD/profiles/
+
+ - name: "test browser crash restart"
+ run: |
+ . venv_tests/bin/activate
+ MOZ_LOG=console:5 python3 test_felt_browser_crash_restart.py ${{ steps.unpack.outputs.runtime_path }} $PWD/geckodriver $PWD/profiles/
+
- name: "test browser safe mode"
run: |
. venv_tests/bin/activate
@@ -554,6 +564,18 @@ jobs:
$Env:MOZ_LOG="console:5"
python3 test_felt_browser_fxa.py "${{ steps.unpack.outputs.firefox_path }}" "${{ steps.unpack.outputs.geckodriver_path }}" "${{ steps.unpack.outputs.profiles_path }}"
+ - name: "test browser crash handled"
+ run: |
+ . venv_tests\Scripts\activate
+ $Env:MOZ_LOG="console:5"
+ python3 test_felt_browser_crash_abort_restart.py "${{ steps.unpack.outputs.firefox_path }}" "${{ steps.unpack.outputs.geckodriver_path }}" "${{ steps.unpack.outputs.profiles_path }}"
+
+ - name: "test browser crash restart"
+ run: |
+ . venv_tests\Scripts\activate
+ $Env:MOZ_LOG="console:5"
+ python3 test_felt_browser_crash_restart.py "${{ steps.unpack.outputs.firefox_path }}" "${{ steps.unpack.outputs.geckodriver_path }}" "${{ steps.unpack.outputs.profiles_path }}"
+
### This is failing on our GCP workers in a non debuggable way.
### GitHub Actions Windows workers are fine and TaskCluster.
# - name: "test browser safe mode"
@@ -993,6 +1015,16 @@ jobs:
. venv_tests/bin/activate
MOZ_LOG=console:5 python3 test_felt_browser_fxa.py "${{ steps.attach.outputs.runtime_path }}" $PWD/geckodriver $PWD/profiles/
+ - name: "test browser crash handled"
+ run: |
+ . venv_tests/bin/activate
+ MOZ_LOG=console:5 python3 test_felt_browser_crash_abort_restart.py "${{ steps.attach.outputs.runtime_path }}" $PWD/geckodriver $PWD/profiles/
+
+ - name: "test browser crash restart"
+ run: |
+ . venv_tests/bin/activate
+ MOZ_LOG=console:5 python3 test_felt_browser_crash_restart.py "${{ steps.attach.outputs.runtime_path }}" $PWD/geckodriver $PWD/profiles/
+
- name: "test browser safe mode"
run: |
. venv_tests/bin/activate
diff --git a/browser/components/enterprise/modules/ConsoleClient.sys.mjs b/browser/components/enterprise/modules/ConsoleClient.sys.mjs
index 156c5a13d3b2b..1bb5b710cc25c 100644
--- a/browser/components/enterprise/modules/ConsoleClient.sys.mjs
+++ b/browser/components/enterprise/modules/ConsoleClient.sys.mjs
@@ -326,7 +326,7 @@ export const ConsoleClient = {
}
const text = await res.text().catch(() => "");
- throw new Error(`Fetch failed (${res.status}): ${text}`);
+ throw new Error(`Fetch ${method} ${path} failed (${res.status}): ${text}`);
},
/**
diff --git a/browser/extensions/felt/api.js b/browser/extensions/felt/api.js
index 42620bc470495..163b39965a57f 100644
--- a/browser/extensions/felt/api.js
+++ b/browser/extensions/felt/api.js
@@ -144,13 +144,12 @@ this.felt = class extends ExtensionAPI {
);
break;
- case "FeltParent:FirefoxAbnormalExit":
- Services.ppmm.removeMessageListener(
- "FeltParent:FirefoxAbnormalExit",
- this
- );
- // TODO: What should we do, restart Firefox?
+ case "FeltParent:FirefoxAbnormalExit": {
+ const success = Services.felt.makeBackgroundProcess(false);
+ console.debug(`FeltExtension: makeBackgroundProcess? ${success}`);
+ this.showWindow("felt-browser-error-multiple-crashes");
break;
+ }
case "FeltParent:FirefoxLogoutExit": {
const success = Services.felt.makeBackgroundProcess(false);
@@ -188,13 +187,17 @@ this.felt = class extends ExtensionAPI {
}
}
- showWindow() {
+ showWindow(errorMessage = "") {
// Height and width are for now set to fit the sso.mozilla.com without the need to resize the window
let flags =
"chrome,private,centerscreen,titlebar,resizable,width=727,height=744";
+ const queryString = errorMessage
+ ? `?error=${encodeURIComponent(errorMessage)}`
+ : "";
+
this._win = Services.ww.openWindow(
null,
- "chrome://felt/content/felt.xhtml",
+ `chrome://felt/content/felt.xhtml${queryString}`,
"_blank",
flags,
null
diff --git a/browser/extensions/felt/content/FeltProcessParent.sys.mjs b/browser/extensions/felt/content/FeltProcessParent.sys.mjs
index 5fb9eeb9a8fc9..c4db6711eb2ea 100644
--- a/browser/extensions/felt/content/FeltProcessParent.sys.mjs
+++ b/browser/extensions/felt/content/FeltProcessParent.sys.mjs
@@ -55,6 +55,26 @@ export class FeltProcessParent extends JSProcessActorParent {
// Track extension ready state (extension must register its observer)
this.extensionReady = false;
+ this.abnormalExitCounter = 0;
+
+ // Amount of abnormal exit to allow over abnormal_exit_period
+ this.abnormalExitLimit = Services.prefs.getIntPref(
+ "enterprise.browser.abnormal_exit_limit",
+ 3
+ );
+
+ /* Time period (in seconds) considered for checking the amount of abnormal
+ * exits. Hitting the limit defined above within this period will stop
+ * automatic restart and show user an error.
+ *
+ * confere shouldAbortRestarting()
+ */
+ this.abnormalExitPeriod = Services.prefs.getIntPref(
+ "enterprise.browser.abnormal_exit_period",
+ 120
+ );
+ this.abnormalExitFirstTime = 0;
+
this.restartObserver = {
observe(aSubject, aTopic) {
console.debug(`FeltExtension: ParentProcess: Received ${aTopic}`);
@@ -253,21 +273,65 @@ export class FeltProcessParent extends JSProcessActorParent {
);
if (!this.restartReported && !this.logoutReported) {
if (this.proc.exitCode === 0) {
+ this.abnormalExitCounter = 0;
+ this.abnormalExitFirstTime = 0;
Services.cpmm.sendAsyncMessage(
"FeltParent:FirefoxNormalExit",
{}
);
} else {
- Services.cpmm.sendAsyncMessage(
- "FeltParent:FirefoxAbnormalExit",
- {}
- );
+ this.handleRestartAfterAbnormalExit();
}
}
});
});
}
+ /**
+ * Handles the abnormal exit and decides whether to restart the Firefox
+ * again or to inform the user of the set of crashes.
+ */
+ handleRestartAfterAbnormalExit() {
+ if (this.abnormalExitCounter === 0) {
+ this.abnormalExitFirstTime =
+ Services.telemetry.msSinceProcessStart() / 1000;
+ }
+ this.abnormalExitCounter += 1;
+ if (this.shouldAbortRestarting()) {
+ console.debug(
+ "Abort restarting Firefox and inform the user of the crashes."
+ );
+ Services.cpmm.sendAsyncMessage("FeltParent:FirefoxAbnormalExit", {});
+ } else {
+ console.debug("Trying to restart Firefox again.");
+ this.startFirefox([]);
+ }
+ }
+ /**
+ * Checks the state of the recent abnormal exits, meaning whether the crashes
+ * counter exceeds a pre-set counter limit within a pre-set time period.
+ *
+ * @returns {boolean} Whether these "abnormal" thresholds are exceeded.
+ */
+ shouldAbortRestarting() {
+ console.debug(
+ `Firefox AbnormalExit abnormalExitLimit=${this.abnormalExitLimit} abnormalExitCounter=${this.abnormalExitCounter} ; firstTime=${this.abnormalExitFirstTime} abnormalExitPeriod=${this.abnormalExitPeriod}`
+ );
+ // Have we reached the limit of allowed crashes ?
+ const isExceedingCrashCounterLimit =
+ this.abnormalExitCounter >= this.abnormalExitLimit;
+ // How much time since the first crash we recorded in this session ?
+ const timeSinceFirstCrash =
+ Services.telemetry.msSinceProcessStart() / 1000 -
+ this.abnormalExitFirstTime;
+ // Is the time since first crash too recent ?
+ const isWithinCrashPeriod = timeSinceFirstCrash <= this.abnormalExitPeriod;
+ console.debug(
+ `Firefox AbnormalExit crashLimitHit=${isExceedingCrashCounterLimit} timeSinceFirstCrash=${timeSinceFirstCrash} crashedNotLongAgoEnough=${isWithinCrashPeriod}`
+ );
+ return isExceedingCrashCounterLimit && isWithinCrashPeriod;
+ }
+
async startFirefoxProcess() {
let socket = Services.felt.oneShotIpcServer();
diff --git a/browser/extensions/felt/content/felt.xhtml b/browser/extensions/felt/content/felt.xhtml
index dbf5ff2135a92..c458716dcbeed 100644
--- a/browser/extensions/felt/content/felt.xhtml
+++ b/browser/extensions/felt/content/felt.xhtml
@@ -81,6 +81,13 @@
+
+
+
+
0)
+ return True
+
+ def test_felt_4_crash_parent_twice(self, exp):
+ self._manually_closed_child = True
+ self.crash_parent()
+ return True
+
+ def test_felt_5_check_error_message(self, exp):
+ self.await_felt_auth_window()
+ self.force_window()
+
+ self._driver.set_context("chrome")
+ self._logger.info("Checking for error message")
+
+ error_msg = self.get_elem(".felt-browser-error-multiple-crashes")
+ assert "crashed multiple times" in error_msg.text, "Error message about crashes"
+
+ return True
+
+
+if __name__ == "__main__":
+ BrowserCrashAbortRestart(
+ "felt_browser_crash_abort_restart.json",
+ firefox=sys.argv[1],
+ geckodriver=sys.argv[2],
+ profile_root=sys.argv[3],
+ env_vars={"MOZ_FELT_UI": "1", "MOZ_GDB_SLEEP": "1"},
+ test_prefs=[
+ ["enterprise.browser.abnormal_exit_limit", 2],
+ ["enterprise.browser.abnormal_exit_period", 120],
+ ],
+ )
diff --git a/testing/enterprise/test_felt_browser_crash_restart.py b/testing/enterprise/test_felt_browser_crash_restart.py
new file mode 100644
index 0000000000000..58dafbdb718d7
--- /dev/null
+++ b/testing/enterprise/test_felt_browser_crash_restart.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import sys
+
+from felt_browser_crashes import BrowserCrashes
+
+
+class BrowserCrashRestart(BrowserCrashes):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def test_felt_2_crash_parent_once(self, exp):
+ self._manually_closed_child = True
+ self.connect_and_crash()
+ return True
+
+ def test_felt_3_proper_restart(self, exp):
+ self._manually_closed_child = False
+ self.wait_process_exit()
+ self._logger.info("Connecting to new browser")
+ self.connect_child_browser()
+ self._browser_pid = self._child_driver.capabilities["moz:processID"]
+ self._logger.info(f"Connected to {self._browser_pid}")
+ self.open_tab_child("about:support")
+
+ version_box = self.get_elem_child("#version-box")
+ self._child_wait.until(lambda d: len(version_box.text) > 0)
+ return True
+
+ def test_felt_4_crash_parent_twice(self, exp):
+ self._manually_closed_child = True
+ self.crash_parent()
+ return True
+
+ def test_felt_5_proper_restart_again(self, exp):
+ self._manually_closed_child = False
+ self.wait_process_exit()
+ self._logger.info("Connecting to new browser")
+ self.connect_child_browser()
+ self._browser_pid = self._child_driver.capabilities["moz:processID"]
+ self._logger.info(f"Connected to {self._browser_pid}")
+ self.open_tab_child("about:support")
+
+ version_box = self.get_elem_child("#version-box")
+ self._child_wait.until(lambda d: len(version_box.text) > 0)
+ return True
+
+
+if __name__ == "__main__":
+ BrowserCrashRestart(
+ "felt_browser_crash_restart.json",
+ firefox=sys.argv[1],
+ geckodriver=sys.argv[2],
+ profile_root=sys.argv[3],
+ env_vars={"MOZ_FELT_UI": "1", "MOZ_GDB_SLEEP": "1"},
+ test_prefs=[
+ ["enterprise.browser.abnormal_exit_limit", 2],
+ ["enterprise.browser.abnormal_exit_period", 1],
+ ],
+ )
diff --git a/testing/enterprise/test_felt_browser_restart_works.py b/testing/enterprise/test_felt_browser_restart_works.py
index 4d5605a968cf5..975c6bc480d8b 100644
--- a/testing/enterprise/test_felt_browser_restart_works.py
+++ b/testing/enterprise/test_felt_browser_restart_works.py
@@ -3,11 +3,8 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-import os
import sys
-import time
-import psutil
from felt_tests import FeltTests
from selenium.common.exceptions import NoSuchWindowException, WebDriverException
@@ -41,30 +38,7 @@ def test_felt_3_perform_restart(self, exp):
return True
def test_felt_4_restart_new_process(self, exp):
- self._logger.info("Waiting a few seconds ...")
- if sys.platform == "win32":
- time.sleep(8)
- else:
- time.sleep(3)
- self._logger.info(f"Checking PID {self._browser_pid}")
-
- if not psutil.pid_exists(self._browser_pid):
- self._logger.info(f"No more PID {self._browser_pid}")
- else:
- try:
- process = psutil.Process(pid=self._browser_pid)
- process_name = process.name()
- process_exe = process.exe()
- process_basename = os.path.basename(process_name)
- process_cmdline = process.cmdline()
- self._logger.info(
- f"Found PID {self._browser_pid}: EXE:{process_exe} :: NAME:{process_name} :: CMDLINE:{process_cmdline} :: BASENAME:'{process_basename}'"
- )
- assert process_basename != "firefox", "Process is not Firefox"
- except psutil.ZombieProcess:
- self._logger.info(f"Zombie found as {self._browser_pid}")
- return True
-
+ self.wait_process_exit()
self._logger.info("Connecting to new browser")
self.connect_child_browser()
new_browser_pid = self._child_driver.capabilities["moz:processID"]