Add auto-healing SCM options with progressive cooling off

IntelPresentMon/PMInstallerLib/Library.wxs

@@ -74,7 +74,33 @@
                   Impersonate="no"
                   Return="ignore" />
 
+    <!-- Crash recovery: after 2s, 15s, 60s, then 5m, start the service again, then stay stopped.
+         The action is "run", not "restart". Restart does not take arguments. The failure command is
+         the single string SCM runs. %1% is the failure-count placeholder SCM substitutes, the same
+         placeholder the Recovery tab checkbox documents. The text in front of it is our option
+         recovery-fail-count, then sc.exe start passes that through to the service.
+         The command is static; the service does not rewrite it.
+         The empty action (//0) is required. If the last action is run, the SCM repeats it forever.
+         reset= 1800 clears the failure count after 30 minutes with no failure; it does not start the service.
+         failureflag 0 counts only abnormal process death, not a clean SERVICE_STOPPED.
+         sc.exe is invoked directly so cmd does not expand %1%. -->
+    <CustomAction Id="ConfigureSharedServiceRecovery"
+                  Directory="SystemFolder"
+                  ExeCommand='sc.exe failure PresentMonSharedService reset= 1800 actions= run/2000/run/15000/run/60000/run/300000//0 command= "sc.exe start PresentMonSharedService --recovery-fail-count %1%"'
+                  Execute="deferred"
+                  Impersonate="no"
+                  Return="ignore" />
+    <CustomAction Id="ConfigureSharedServiceRecoveryFlag"
+                  Directory="SystemFolder"
+                  ExeCommand='sc.exe failureflag PresentMonSharedService 0'
+                  Execute="deferred"
+                  Impersonate="no"
+                  Return="ignore" />
+
     <InstallExecuteSequence>
+        <!-- Configure recovery before the service is started. -->
+        <Custom Action="ConfigureSharedServiceRecovery" After="InstallServices">NOT REMOVE</Custom>
+        <Custom Action="ConfigureSharedServiceRecoveryFlag" After="ConfigureSharedServiceRecovery">NOT REMOVE</Custom>
         <!-- After InstallServices has registered the service (and StartServices would have run) -->
         <Custom Action="TryStartSharedService" After="StartServices">1</Custom>
     </InstallExecuteSequence>

IntelPresentMon/PresentMonService/CliOptions.h

@@ -26,6 +26,7 @@ namespace clio
 	private: Group gd_{ this, "Debugging", "Aids in debugging this tool" }; public:
 		Flag debug{ this, "--debug,-d", "Stall service by running in a loop after startup waiting for debugger to connect" };
 		Option<long long> timedStop{ this, "--timed-stop", -1, "Signal stop event after specified number of milliseconds" };
+		Option<uint32_t> recoveryFailCount{ this, "--recovery-fail-count", 0, "Failure count SCM passes when it starts the service on recovery" };
 
 	private: Group gr_{ this, "Playback", "Playback of recorded ETL files" }; public:
 		Option<std::string> etlTestFile{ this, "--etl-test-file", "", "Etl test file including necessary path", CLI::ExistingFile };

IntelPresentMon/PresentMonService/Service.cpp

@@ -5,7 +5,6 @@
 #include <assert.h>
 #include <strsafe.h>
 #include <dbt.h>
-#include <format>
 #include "PMMainThread.h"
 #include <iostream>
 #include <chrono>
@@ -134,7 +133,7 @@ void ConcreteService::ServiceInit()
 		FALSE,          // auto reset event
 		FALSE,          // not signaled
 		NULL);          // no name
-	if (mServiceStopEventHandle == nullptr) {
+	if (mResetPowerTelemetryEventHandle == nullptr) {
 		ReportServiceStatus(SERVICE_STOPPED, GetLastError(), 0);
 		return;
 	}

IntelPresentMon/PresentMonService/ServiceMain.cpp

@@ -50,6 +50,10 @@ int CommonEntry(DWORD argc, LPTSTR* argv, bool asApp)
 		log::GetLevelName(log::GlobalPolicy::Get().GetLogLevel()),
 		log::GetLevelName(PMLOG_BUILD_LEVEL_)));
 
+	if (const auto failCount = clio::Options::Get().recoveryFailCount.AsOptional()) {
+		pmlog_warn(std::format("SCM recovery start, fail count {}.", *failCount)).no_trace();
+	}
+
 	if (asApp) {
 		auto& svc = ConsoleDebugMockService::Get();
 		svc.Run();

README-Service.md

@@ -18,4 +18,22 @@ PresentMonAPI.h, PresentMonAPI2Loader.lib, and PresentMonAPI2Loader.dll are opti
 
 ## Diagnostics
 
-All of the PresentMonAPI functions return an enum type PM_STATUS that indicates success/failure and can give a hint as to the cause of any failure. For more detailed diagnostic messages, refer to PresentMonDiagnostics.h found in the PresentMonAPI2 project directory.
+All of the PresentMonAPI functions return an enum type PM_STATUS that indicates success/failure and can give a hint as to the cause of any failure. For more detailed diagnostic messages, refer to PresentMonDiagnostics.h found in the PresentMonAPI2 project directory.
+
+# Failure recovery
+
+The installer configures `PresentMonSharedService` so Windows starts it again after an abnormal process exit, then stops trying. The Recovery action is "Run a program", not "Restart the Service". Restart does not take a command line. The program is `sc.exe start PresentMonSharedService --recovery-fail-count %1%`. `%1%` is the failure-count placeholder SCM substitutes when it runs the command (the same placeholder the Recovery tab checkbox documents as `/fail=%1%`). `sc.exe start` passes `--recovery-fail-count N` through to the service. The command string is fixed at install time. The service does not register or rewrite it.
+
+| Failure in the window | Action | Holdoff |
+| --- | --- | --- |
+| 1st | start with `--recovery-fail-count 1` | 2 seconds |
+| 2nd | start with `--recovery-fail-count 2` | 15 seconds |
+| 3rd | start with `--recovery-fail-count 3` | 60 seconds |
+| 4th | start with `--recovery-fail-count 4` | 5 minutes |
+| 5th and later | stay stopped | |
+
+The failure count resets after 30 minutes with no failure. That reset does not start the service. A later start gets a fresh budget only after those 30 minutes have elapsed. A clean stop (`sc.exe stop`) does not count as a failure.
+
+On a recovery start the service log contains `SCM recovery start, fail count N`. A normal start has no `--recovery-fail-count` argument and does not log that line.
+
+An existing client session is not reconnected. The next API call on that handle returns `PM_STATUS_SESSION_NOT_OPEN`. Opening a new session after the service is back is the caller's responsibility.