Topic: How to reboot a rig if it stops mining? (Read 11952 times)

$LogFile     = "MiningMonitor.log"
$verboseLogInfo = $false
$ProcessName = "openhardwaremonitor"
$Date        = Get-Date
$tooLow      = 40     # This is the Celsius temperature you wish to "flag" as an offline GPU
$reportIftooHigh = $true
$tooHigh     = 60           # This temperature triggers a warning text
$wayTooHigh  = 82           # This temperature and above shuts down the miner if reached on ONE occasion
$lowestReportedTemp  = 1000 # Remember to check that HardwareMonitor is reporting in Celsius
$highestReportedTemp = 0
$numIterations = 9          # Set the number of iterations to test before considering a "failed" miner process (too low).
$rebootIfLow = $true        # Set to true to send text and reboot (production) when low temps are found. Set to false to only send text.

# Textual elements
$textBodyReboot = "At $Date, Miner rebooted - GPU temperature dropped to:"
$textBodyWarning = "At $Date, GPU temperature dropped to:"
$textBodyHotWarning = "At $Date, a GPU reached warning temp of:"

# Constants for email message
$senderEmail = '[email protected]'
$senderPassword = 'yourpassword'
# you must set this for your phone number and carrier. The list is here: http://www.emailtextmessages.com/
$textNumberAndCarrier = '[email protected]'
$textSubject =  'Miner Message';
$smtpServer = 'smtp.mail.yahoo.com'
$port = '587'
$useSsl = $true


if ($rebootIfLow) {
    Write-Host "Monitor will send Message and Reboot if any GPU is consistently below $tooLow"
    $textBody = $textBodyReboot
    }
    else {
    Write-Host "Monitor will send Text Message if any GPU is consistently below $tooLow"
    $textBody = $textBodyWarning
    }
if ($reportIftooHigh) { Write-Host "Monitoring for hot GPUs. Message sent at $tooHigh, Reboot if any GPU reaches $wayTooHigh" }

# [console]::TreatControlCAsInput = $true
try
{
#Test if openhardwaremonitor is running and if not, start it
if((get-process $ProcessName -ErrorAction SilentlyContinue) -eq $Null)
    { Start-Process -FilePath ".\OpenHardwareMonitor.exe" -WorkingDirectory ".\"} # -WindowStyle Minimized; echo "Starting OpenHardwareMonitor..." }
elseif ($verboseLogInfo )
    { echo "OpenHardwareMonitor is running" }

$countOfIterationsLow = 0
$countIterationsHigh  = 0
$hotGPU = 99
#if the computer just started or OpenHardwareMonitor is slow starting we will have problems, so insert wait here
Start-Sleep 90

For ($i=1; $i -le $numIterations; $i++) {
# Query GPU temperature from OpenHardwareMonitor
$GPUTempObj = Get-WmiObject -namespace root\openhardwaremonitor -class sensor | Where-Object {$_.SensorType -Match "temperature" -and $_.Identifier -like "*gpu*"}
    $GPU_Num = 0;
    ForEach($GPU In $GPUTempObj)
{
$GPUTemp = $GPU.value
if($GPUTemp -lt $tooLow)
{
Write-Host "GPU$GPU_Num is $GPUTemp, below cutoff of $tooLow"
"$Date - GPU$GPU_Num is $GPUTemp, below cutoff of $tooLow" | Out-File $LogFile -Append
$countOfIterationsLow = $countOfIterationsLow + 1
                if ($GPUTemp -lt $lowestReportedTemp) { $lowestReportedTemp = $GPUTemp }
}
  else
{
Write-Host "GPU$GPU_Num is OK at $GPUTemp degrees (above $tooLow)"   
}
            $GPU_Num++
}
         if ($reportIftooHigh) {
             if($GPUTemp -ge $tooHigh)
    {
    Write-Host "GPU$GPU_Num is $GPUTemp, hotter than the cutoff of $tooHigh degrees"
    "$Date - GPU$GPU_Num is $GPUTemp, hotter than cutoff of $tooHigh" | Out-File $LogFile -Append
     $countOfIterationsHigh++
                    if ($GPUTemp -gt $highestReportedTemp) { $highestReportedTemp = $GPUTemp; $hotGPU = $GPU_Num }
    }
              if ($GPUTemp -ge $wayTooHigh) { Stop-Computer -force } 
         }
         if ($Host.UI.RawUI.KeyAvailable -and (3 -eq [int]$Host.UI.RawUI.ReadKey("AllowCtrlC,IncludeKeyUp,NoEcho").Character))
            {
                Write-Host "You pressed CTRL-C. Do you want to continue Mining Monitor (Y/N)?"
                $key = $Host.UI.RawUI.ReadKey("NoEcho, IncludeKeyDown")
                if ($key.Character -eq "N") { break; }
            }   
           
#if we have bad timing on a driver crash (and recovery) or work restarts we may get low results, so we wait between tests
if ($i -lt $numIterations) {
        Write-Host "- - - - - - - - - - -"
Start-Sleep 20
}
    }

# All tests (determined by &numIterations) have to get a low result restart
if($countOfIterationsLow -ge $numIterations)
     {"$Date - $textBody $lowestReportedTemp on $countOfIterationsLow occasions with cutoff of $tooLow - restarting" | Out-File $LogFile -Append
$MailArgs = @{
From       = $senderEmail
To         = $textNumberAndCarrier
Subject    = $textSubject
Body       = "$textBody $lowestReportedTemp on $countOfIterationsLow occasions with cutoff of $tooLow"
SmtpServer = $smtpServer
Port       = $port
UseSsl     = $useSsl
Credential = New-Object pscredential $senderEmail,$($senderPassword |ConvertTo-SecureString -AsPlainText -Force)
}
Send-MailMessage @MailArgs
if ($rebootIfLow) {
Restart-Computer -force
}
     }
elseif ($lowestReportedTemp -lt $tooLow)
     {"$Date - $countOfIterationsLow results below cutoff, lowest was $lowestReportedTemp with report cutoff of $tooLow" | Out-File $LogFile -Append}
elseif ($verboseLogInfo)
    {"$Date - No concerning results, cutoff temperature currently: $tooLow" | Out-File $LogFile -Append}


# Some tests (determined by &numIterations) reported GPUs too high
if($countOfIterationsHigh -ge $numIterations) {
     "$Date - GPU$hotGPU reported $highestReportedTemp degrees $countOfIterationsHigh times with warning set to $tooHigh" | Out-File $LogFile -Append
$MailArgs = @{
From       = $senderEmail
To         = $textNumberAndCarrier
Subject    = "Hot GPU - $textSubject"
Body       = "$Date - GPU$hotGPU reached $highestReportedTemp degrees $countOfIterationsHigh times. Current warning set to $tooHigh"
SmtpServer = $smtpServer
Port       = $port
UseSsl     = $useSsl
Credential = New-Object pscredential $senderEmail,$($senderPassword |ConvertTo-SecureString -AsPlainText -Force)
}
Send-MailMessage @MailArgs
}
}
finally
{
    if ($verboseLogInfo) { "$Date - Mining Monitor Stopped" | Out-File $LogFile -Append }
}

@echo off
set counter=0
:start
cls
set /a counter=counter+1
echo Mining Monitor loop %counter%
powershell.exe .\MonitorMining.ps1
TIMEOUT /T 30
goto start

$LogFile     = "MiningMonitor.log"
$ProcessName = "openhardwaremonitor"
$Date        = Get-Date
$TooLow      = 40   # This is the Celsius temperature you wish to "flag" as an offline GPU
$TooHigh     = 70         # This temperature triggers a warning text
$lowestReportedTemp = 1000  # Remember to check that HardwareMonitor is reporting in Celsius
$numIterations = 9        # Set the number of iterations to test before considering a "failed" miner process.
$rebootIfLow = $true     # Set to true to send text and reboot (production) when low temps are found. Set to false to only send text.

# Textual elements
$textBodyReboot = "At $Date, Miner rebooted - GPU temperature dropped to:"
$textBodyWarning = "At $Date, GPU temperature dropped to:"

# Constants for email message
$senderEmail = '[email protected]'
$senderPassword = 'emailpassword'
# you must set this for your phone number and carrier. The list is here: http://www.emailtextmessages.com/
$textNumberAndCarrier = '[email protected]'
$textSubject =  'Miner Message';
$smtpServer = 'smtp.mail.yourprovider.com'
$port = '587'
$useSsl = $true


if ($rebootIfLow) {
    Write-Host "Monitor will send Message and Reboot if Needed"
    $textBody = $textBodyReboot
    }
    else {
    Write-Host "Monitor is in Message Only Mode"
    $textBody = $textBodyWarning
    }

# [console]::TreatControlCAsInput = $true
try
{
#Test if openhardwaremonitor is running and if not, start it
if((get-process $ProcessName -ErrorAction SilentlyContinue) -eq $Null)
    { Start-Process -FilePath ".\OpenHardwareMonitor.exe" -WorkingDirectory ".\"} # -WindowStyle Minimized; echo "Starting OpenHardwareMonitor..." }
else
    { echo "OpenHardwareMonitor is running" }

$countOfIterationsLow = 0
#if the computer just started or OpenHardwareMonitor is slow starting we will have problems, so insert wait here
Start-Sleep 120

For ($i=1; $i -le $numIterations; $i++) {
# Query GPU temperature from OpenHardwareMonitor
$GPUTempObj = Get-WmiObject -namespace root\openhardwaremonitor -class sensor | Where-Object {$_.SensorType -Match "temperature" -and $_.Identifier -like "*gpu*"}
    $GPU_Num = 0;
    ForEach($GPU In $GPUTempObj)
{
$GPUTemp = $GPU.value
if($GPUTemp -lt $TooLow)
{
Write-Host "GPU$GPU_Num is $GPUTemp, below cutoff of $TooLow"
"$Date - GPU$GPU_Num is $GPUTemp, below cutoff of $TooLow" | Out-File $LogFile -Append
$countOfIterationsLow = $countOfIterationsLow + 1
                if ($GPUTemp -lt $lowestReportedTemp) { $lowestReportedTemp = $GPUTemp }
}
  else
{
Write-Host "GPU$GPU_Num is OK at $GPUTemp degrees (above $TooLow)"   
}
            $GPU_Num++
}
         if ($Host.UI.RawUI.KeyAvailable -and (3 -eq [int]$Host.UI.RawUI.ReadKey("AllowCtrlC,IncludeKeyUp,NoEcho").Character))
            {
                Write-Host "You pressed CTRL-C. Do you want to continue Mining Monitor (Y/N)?"
                $key = $Host.UI.RawUI.ReadKey("NoEcho, IncludeKeyDown")
                if ($key.Character -eq "N") { break; }
            }   
           
#if we have bad timing on a driver crash (and recovery) or work restarts we may get low results, so we wait between tests
if ($i -lt $numIterations) {
        Write-Host "- - - - - - - - - - -"
Start-Sleep 20
}
    }

# All tests (determined by &numIterations) have to get a low result restart
if($countOfIterationsLow -ge $numIterations)
     {"$Date - $textBody $lowestReportedTemp on $countOfIterationsLow occasions with cutoff of $TooLow - restarting" | Out-File $LogFile -Append
$MailArgs = @{
From       = $senderEmail
To         = $textNumberAndCarrier
Subject    = $textSubject
Body       = "$textBody $lowestReportedTemp on $countOfIterationsLow occasions with cutoff of $TooLow"
SmtpServer = $smtpServer
Port       = $port
UseSsl     = $useSsl
Credential = New-Object pscredential $senderEmail,$($senderPassword |ConvertTo-SecureString -AsPlainText -Force)
}
Send-MailMessage @MailArgs
if ($rebootIfLow) {
Restart-Computer -force
}
     }
elseif ($lowestReportedTemp -lt $TooLow)
     {"$Date - $countOfIterationsLow results below cutoff, lowest was $lowestReportedTemp with report cutoff of $TooLow" | Out-File $LogFile -Append}
else
    {"$Date - No concerning results, cutoff temperature currently: $TooLow" | Out-File $LogFile -Append}
}

finally
{
    "$Date - Mining Monitor Stopped" | Out-File $LogFile -Append
}   

@echo off
set counter=0
:start
cls
set /a counter=counter+1
echo Mining Monitor loop %counter%
powershell.exe .\MonitorMining.ps1
TIMEOUT /T 30
goto start

   { Start-Process -FilePath ".\OpenHardwareMonitor.exe" -WorkingDirectory ".\"} # -WindowStyle Minimized; echo "Starting OpenHardwareMonitor..." }