This a two node hyper-converged cluster that I abused by moving ethernet cables between two connected ethernet switches while it was running. It went into synchronization. When the dust cleared, and it finished resynchronizing, the cluster was down. Here's what I can't figure out how to resolve.
One of the servers says all devices are active (state 0), synchronized (status 1) with synch percent 100%.
The other server is the same (i.e., synchronized), except that synch percent is zero (0%).
What's weird is that the CSV cluster shared volumes appear to be synchronized, and synch is active. I can put test files on one server and they show up on the other immediately. A quick check shows the same number of bytes used on both servers. But my Windows Failover Cluster Manager is having none of it. Not happy at all.
I've checked my iSCSI targets, all looks good there, too -- connected both ways.
I've resisted the temptation to try to force synchronized status -- which doesn't really make sense to do since they both do claimed to be synchronized on the basis of status, if not percent.
I'm attaching a copy of the latest version of my customized status script and its output. If you like the script, feel free to use it: it makes several things easier.
Like I say, I can't find any discussion or documentation that covers this situation. Help!
Here's the output from both servers:
Code: Select all
Server HV3
------------- Running ---------------
DeviceName: HAImage1 Sync Status: 1 -- TargetName: iqn.2008-08.com.starwindsoftware:kmhv3.kmsi.net-witness
Device state: 0 (Active)
Synchronized: 100%
ha_partner_nodes_count: 1
ha_partner_node1_sync_status: 1
ha_partner_node1_sync_percent: 0
ha_partner_node1_is_exist_sync_valid_connection: 1
ha_partner_node1_is_exist_heartbeat_valid_connection: 1
DeviceName: HAImage2 Sync Status: 1 -- TargetName: iqn.2008-08.com.starwindsoftware:kmhv3-csv1
Device state: 0 (Active)
Synchronized: 100%
ha_partner_nodes_count: 1
ha_partner_node1_sync_status: 1
ha_partner_node1_sync_percent: 0
ha_partner_node1_is_exist_sync_valid_connection: 1
ha_partner_node1_is_exist_heartbeat_valid_connection: 1
Server HV4
------------- Running ---------------
DeviceName: HAImage1 Sync Status: 1 -- TargetName: iqn.2008-08.com.starwindsoftware:kmhv4-witness
Device state: 0 (Active)
Synchronized: 0%
ha_partner_nodes_count: 1
ha_partner_node1_sync_status: 1
ha_partner_node1_sync_percent: 100
ha_partner_node1_is_exist_sync_valid_connection: 1
ha_partner_node1_is_exist_heartbeat_valid_connection: 1
DeviceName: HAImage2 Sync Status: 1 -- TargetName: iqn.2008-08.com.starwindsoftware:kmhv4.kmsi.net-csv1
Device state: 0 (Active)
Synchronized: 0%
ha_partner_nodes_count: 1
ha_partner_node1_sync_status: 1
ha_partner_node1_sync_percent: 100
ha_partner_node1_is_exist_sync_valid_connection: 1
ha_partner_node1_is_exist_heartbeat_valid_connection: 1Code: Select all
#
# This following example shows how to get synchronization status of specified HA device and
# if there is a need run synchronization
#
Import-Module StarWindX
while ($true) {
"------------- Running ---------------"
try
{
#
# connect to the server
#
$server = New-SWServer -host 127.0.0.1 -port 3261 -user root -password starwind
$server.Connect()
#
# Try to find specified device
#
$deviceName = "*"
$partnerTargetName = "*"
$deviceFound = $false
foreach($device in $server.devices)
{
Write-Host ""
Write-Host -NoNewline "DeviceName:" $device.GetPropertyValue("DeviceName") -foreground yellow
$device.Refresh()
$syncState = $device.GetPropertyValue("ha_synch_status")
Write-Host -NoNewline "Sync State: " $syncState
If ( $syncState -ne "" )
{
Write-Host " -- TargetName:" $device.GetPropertyValue("TargetName") -foreground yellow
$state = $device.GetPropertyValue("state")
Write-Host -NoNewline "Device state:" $device.GetPropertyValue("state")
switch ($state)
{
0 { " (Active)" }
1 { " (NonActive)" }
2 { " (NotLicensed)" }
3 { " (Disabled)" }
default { "(Undefined)" }
}
$waitForAutoSync = $device.GetPropertyValue("ha_wait_on_autosynch")
if ( $waitForAutoSync -eq "1" )
{
Write-Host "Waiting for autosynchronization..." -foreground yellow
}
else
{
if ( $syncState -eq "1" )
{
#
# Device is synchronized. Get synchronization percent and show it
#
$syncPercent = $device.GetPropertyValue("ha_synch_percent")
Write-Host "Synchronized: $($syncPercent)%" -foreground yellow
}
if ( $syncState -eq "2" )
{
#
# Device is synchronizing. Get synchronization percent and show it
#
$syncPercent = $device.GetPropertyValue("ha_synch_percent")
Write-Host "Synchronizing: $($syncPercent)%" -foreground yellow
}
if ( $syncState -eq "3" )
{
#
# Device not synchronized. Synchronize current node from partner
#
Write-Host "Device not synchronized."
# Synchronize current node from partner '$($partnerTargetName)'" -foreground yellow
# $params = new-object -ComObject StarWindX.Parameters
# $params.AppendParam("deviceID",$device.DeviceId)
# $params.AppendParam("partnetTargetName",$partnerTargetName)
# $server.ExecuteCommand( 0, "restoreHAPartnerNode", $params)
#
# If you want to synchronize partners from current node you can comment out code above and uncomment section below
#
# Device not synchronized. Mark current node as 'Synchronized'.
# WARNING, Command changes Device Status to "Synchronized" without Data Synchronization with HA (High Availability) Partner,
# Device will start processing Client Requests immediately and will be used as Data Synchronization Source for Partner Device.
#
#Write-Host "Device not synchronized. Mark current node as 'Synchronized'. " -foreground yellow
#$params = new-object -ComObject StarWindX.Parameters
#$params.AppendParam("deviceID",$device.DeviceId)
#$server.ExecuteCommand( 0, "restoreCurrentHANode", $params)
# Start-Sleep -m 5000
}
}
#=======
Write-Host "ha_partner_nodes_count:" $device.GetPropertyValue("ha_partner_nodes_count")
Write-Host "ha_partner_node1_sync_status:" $device.GetPropertyValue("ha_partner_node1_sync_status")
Write-Host "ha_partner_node1_sync_percent:" $device.GetPropertyValue("ha_partner_node1_sync_percent")
Write-Host "ha_partner_node1_is_exist_sync_valid_connection:" $device.GetPropertyValue("ha_partner_node1_is_exist_sync_valid_connection")
Write-Host "ha_partner_node1_is_exist_heartbeat_valid_connection:" $device.GetPropertyValue("ha_partner_node1_is_exist_heartbeat_valid_connection")
# Write-Host "ha_partner_node2_sync_status:" $device.GetPropertyValue("ha_partner_node2_sync_status")
# Write-Host "ha_partner_node2_sync_percent:" $device.GetPropertyValue("ha_partner_node2_sync_status")
}
else
{
write-host -NoNewLine " -- no info"
}
}
# if ( $deviceFound -ne $true )
# {
# Write-Host "$($deviceName) not found" -foreground red
# }
}
catch
{
Write-Host "Exception $($_.Exception.Message)" -foreground red
}
Write-Host ""
$server.Disconnect( )
pause
}