Category: misc

#! /usr/bin/pwsh
$user = '[email protected]'
# Import password from an encrypted file
$encryptedPassword = Import-Clixml -Path '/glabs/spec/vcsa_admin.xml'
$decryptedPassword = $encryptedPassword.GetNetworkCredential().Password



# Function to check if vCenter services are running
function Test-VCenterServicesRunning {
    $serviceInstance = Connect-VIServer -Server vcsa01.glabs.local -Username  $user -Password  $decryptedPassword -ErrorAction SilentlyContinue
    
    if ($serviceInstance -eq $null) {
        return $false
    }
    
    $serviceContent = Get-View -Id $serviceInstance.ExtensionData.content.ServiceInstance
    
    $serviceContent.serviceInfo.service | ForEach-Object {
        if ($_.running -eq $false) {
            Disconnect-VIServer -Server $vcServer -Confirm:$false
            return $false
        }
    }
    
    Disconnect-VIServer -Server $vcServer -Confirm:$false
    return $true
}

# Wait for vCenter services to start
Write-Host "Waiting for vCenter services to start..."

while (-not (Test-VCenterServicesRunning)) {
    Start-Sleep -Seconds 5
}

Write-Host "vCenter services are running. Connecting to vCenter..."




#connect to vc and add hosts
Connect-viserver vcsa01.glabs.local -User $user -Password $decryptedPassword

#crate datacenter and cluster
New-Datacenter -Location Datacenters  -Name cloud
New-Cluster -Name "management" -Location "cloud"

Add-VMHost -Name esxi01.Glabs.local -Location management -user 'root' -password 'bAdP@$$' -Force -Confirm:$false 
Add-VMHost -Name esxi02.Glabs.local -Location management -user 'root' -password 'bAdP@$$' -Force -Confirm:$false 
Add-VMHost -Name esxi03.Glabs.local -Location management -user 'root' -password 'bAdP@$$' -Force -Confirm:$false 
get-vmhost | Get-VMHostStorage -RescanAllHba -RescanVmfs


$cache = 'mpx.vmhba0:C0:T1:L0'
$data = 'mpx.vmhba0:C0:T2:L0'

#mask cache disk as ssd
$esx = Get-VMHost -Name esxi01.glabs.local
$storSys = Get-View -Id $esx.ExtensionData.ConfigManager.StorageSystem
$uuid = $storSys.StorageDeviceInfo.ScsiLun | where {$_.CanonicalName -eq $cache} 
$storSys.MarkAsSsd($uuid.Uuid)
$esx = Get-VMHost -Name esxi02.glabs.local
$storSys = Get-View -Id $esx.ExtensionData.ConfigManager.StorageSystem
$uuid = $storSys.StorageDeviceInfo.ScsiLun | where {$_.CanonicalName -eq $cache} 
$storSys.MarkAsSsd($uuid.Uuid)
$esx = Get-VMHost -Name esxi03.glabs.local
$storSys = Get-View -Id $esx.ExtensionData.ConfigManager.StorageSystem
$uuid = $storSys.StorageDeviceInfo.ScsiLun | where {$_.CanonicalName -eq $cache} 
$storSys.MarkAsSsd($uuid.Uuid)

#add vSAN service to portgroup
$VMKNetforVSAN = "iscsi_1"
Get-VMHostNetworkAdapter -VMKernel | Where {$_.PortGroupName -eq $VMKNetforVSAN }|Set-VMHostNetworkAdapter -VsanTrafficEnabled $true -Confirm:$false



#Create vSAN cluster
get-cluster management | Set-Cluster -VsanEnabled:$true -VsanDiskClaimMode Manual -Confirm:$false -ErrorAction SilentlyContinue

#wait for previous task to finish
start-sleep 60

#add disk disk groups
New-VsanDiskGroup -VMHost esxi01.glabs.local -SSDCanonicalName $cache -DataDiskCanonicalName $data
New-VsanDiskGroup -VMHost esxi02.glabs.local -SSDCanonicalName $cache -DataDiskCanonicalName $data
New-VsanDiskGroup -VMHost esxi03.glabs.local -SSDCanonicalName $cache -DataDiskCanonicalName $data

#mount nfs 
get-vmhost | New-Datastore -Nfs -Name iso -Path /volume1/iso -NfsHost iso.glabs.local -ReadOnly

#noidea why the above does not work for vsphere7 but running the below manualy on a deployed env preps it for vSAN, dont touch it if it aint broken?
get-cluster management | Set-Cluster -VsanEnabled:$true -VsanDiskClaimMode Manual -Confirm:$false -ErrorAction SilentlyContinue


disconnect-viserver -confirm:$false

VMware powerCli installation fails with the below:

PS C:\Users\Administrator> Install-Module -Name VMware.PowerCLI                                                                                                                                                                                 NuGet provider is required to continue                                                                                  PowerShellGet requires NuGet provider version '2.8.5.201' or newer to interact with NuGet-based repositories. The NuGet  provider must be available in 'C:\Program Files\PackageManagement\ProviderAssemblies' or                               'C:\Users\Administrator\AppData\Local\PackageManagement\ProviderAssemblies'. You can also install the NuGet provider by  running 'Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force'. Do you want PowerShellGet to install   and import the NuGet provider now?                                                                                      [Y] Yes  [N] No  [S] Suspend  [?] Help (default is "Y"): y

Untrusted repository
You are installing the modules from an untrusted repository. If you trust this repository, change its
InstallationPolicy value by running the Set-PSRepository cmdlet. Are you sure you want to install the modules from
'PSGallery'?
[Y] Yes  [A] Yes to All  [N] No  [L] No to All  [S] Suspend  [?] Help (default is "N"): y
PackageManagement\Install-Package : The module 'VMware.VimAutomation.Sdk' cannot be installed or updated because the
authenticode signature of the file 'VMware.VimAutomation.Sdk.cat' is not valid.
At C:\Program Files\WindowsPowerShell\Modules\PowerShellGet\1.0.0.1\PSModule.psm1:1809 char:21
+ ...          $null = PackageManagement\Install-Package @PSBoundParameters
+                      ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    + CategoryInfo          : InvalidOperation: (Microsoft.Power....InstallPackage:InstallPackage) [Install-Package],
   Exception
    + FullyQualifiedErrorId : InvalidAuthenticodeSignature,ValidateAndGet-AuthenticodeSignature,Microsoft.PowerShell.P
   ackageManagement.Cmdlets.InstallPackage

Workaround: install PowerShell by skipping publisher checks:

install-module vmware.powercli -scope AllUsers -force -SkipPublisherCheck -AllowClobber

Cause: This is due to the fact that the certificate we used to sign the modules was replaced with a new one from a new publisher. 

curl –header “Content-Type: application/json” \
–request POST \
–data ‘{“username”:”xyz”,”password”:”xyz”}’ \
https://IP/suite-api/api/auth/token/acquire

adding a webhook to vRA-Code Stream fails with error:

java.lang.IllegalArgumentException: 400 BAD_REQUEST "Unable to create webhook at Git server. Request failed with : 422"

browser console:

{timestamp: 1622639483332, path: "/codestream/api/git-webhooks", status: 400, error: "Bad Request",…}
@type: "com.vmware.codestream.exception.CodestreamException"
error: "Bad Request"
message: "400 BAD_REQUEST \"Unable to create webhook at Git server. Request failed with : 422\""
path: "/codestream/api/git-webhooks"
requestId: "929b7098-102300"
status: 400
timestamp: 1622639483332

Symptoms: Gitlab is installed on the same network as of vRA/Codestream

Cause: Gitlab does not allow connections to create a webhook from local subnet by default.

Resolution :Allow requests to the local network from web hooks and services

Steps:

• On gitlab, go to admin area:

• On the left hand, navigate to settings> Network

• enable Allow requests to the local network from system hooks, also add the codestream FQDN (or *) in the URL
  

Now Go back to codestream and add the webhook: success!!

Usage meter reports partial collection failure: events

Logs files: 	vccol_main.log | vccol_error.log
[2021-05-18 09:02:25]  | ERROR | ter collector thread |    com.vmware.um.vccollector.VCCollector | vCenter collector90 | Events stage raised exception javax.xml.ws.WebServiceException: java.net.SocketTimeoutException: Read timed out java.net.SocketTimeoutException: Read timed out=>Read timed out
[2021-05-18 09:02:26]  | ERROR | ter collector thread | com.vmware.um.collector.CollectionHelper | vCenter collector98 | Status (COLLECT_API_ERR) for vCenter server 7: Partial collection failure: Events
[2021-05-18 10:02:18]  | ERROR | ter collector thread |    com.vmware.um.vccollector.VCCollector | vCenter collector179 | Events stage raised exception javax.xml.ws.WebServiceException: java.net.SocketTimeoutException: Read timed out java.net.SocketTimeoutException: Read timed out=>Read timed out

Cause: Connection was closed before the data could be retrieved successfully. Usage meter requests vCenter for events, this api generally takes some time to respond either due to the huge number of events or slowly due to heavy processing on the vCenter.

Resolution: Increase timeouts
* take a snapshot of the um appliance.
* ssh into the appliance with the user usagemeter
* take a backup copy of common_utils.sh

cp /opt/vmware/cloudusagemetering/scripts/common_utils.sh /opt/vmware/cloudusagemetering/scripts/common_utils.sh.bak

* edit the config file using vi

 vi /opt/vmware/cloudusagemetering/scripts/common_utils.sh

* replace the values of the below field

=> CONNECT_TIMEOUT_MS="300000"
=> READ_TIMEOUT_MS="600000"

* save the file and restart the appliance

Note: On the vCenter side, if there are bursts of events, then this is also a likely scenario. KB https://kb.vmware.com/s/article/74607 is one among several where the burst is documented (event bursts need to be triaged from vCenter prospective)

Ensure that you have set up sphere provider provider, refer my previous blog https://blog.ntitta.in/?p=597

create a windows profile

/etc/salt/cloud.profiles.d/w16k.conf 

root@saltyub:/# cat /etc/salt/cloud.profiles.d/w16k.conf 
w16k:
  provider: vcsa
  clonefrom: w16k_salt 
#  devices: 
#   network: 
#    Network adaptor 1:
#     name: VM Network
#     adapter_type: vmxnet3
#     switch_type: standard
#     ip: 172.16.70.79
#     gateway: [172.16.1.1]
#     subnet_mask: 255.255.128.0
#     domain: ntitta.lab
  cluster: vSAN
  datastore: vsanDatastore
  power_on: True
  deploy: True
  customization: True
  minion:
   master: saltyu.ntitta.lab
  win_username: administrator 
  win_password: 'P@ssw0d'
  plain_text: True
  win_user_fullname: admin
  win_run_once: 'powershell.exe c:\scripts\e.winrm.ps1'
  win_installer: /salt/minion/Salt-Minion-3000.9-Py2-AMD64-Setup.exe
  winrm_verify_ssl: False

Ensure that you have the smbprotocol and pypsexec installed

pip3 install smbprotocol
pip3 install pypsexec

on the guest windows server template, ensure vmware tools is installed and create a PowerShell script in the path: c:\scripts\e.winrm.ps1, refer salt doc for more information: https://docs.saltproject.io/en/latest/topics/cloud/windows.html

New-NetFirewallRule -Name "SMB445" -DisplayName "SMB445" -Protocol TCP -LocalPort 445
New-NetFirewallRule -Name "WINRM5986" -DisplayName "WINRM5986" -Protocol TCP -LocalPort 5986

winrm quickconfig -q
winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="300"}'
winrm set winrm/config '@{MaxTimeoutms="1800000"}'
winrm set winrm/config/service/auth '@{Basic="true"}'

$SourceStoreScope = 'LocalMachine'
$SourceStorename = 'Remote Desktop'

$SourceStore = New-Object -TypeName System.Security.Cryptography.X509Certificates.X509Store -ArgumentList $SourceStorename, $SourceStoreScope
$SourceStore.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadOnly)

$cert = $SourceStore.Certificates | Where-Object -FilterScript {
    $_.subject -like '*'
}

$DestStoreScope = 'LocalMachine'
$DestStoreName = 'My'

$DestStore = New-Object -TypeName System.Security.Cryptography.X509Certificates.X509Store -ArgumentList $DestStoreName, $DestStoreScope
$DestStore.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite)
$DestStore.Add($cert)

$SourceStore.Close()
$DestStore.Close()

winrm create winrm/config/listener?Address=*+Transport=HTTPS `@`{CertificateThumbprint=`"($cert.Thumbprint)`"`}

Restart-Service winrm

download salt windows minion installer to the below path on the salt-master:
/salt/minion/, exe can be downloaded from https://docs.saltproject.io/en/latest/topics/installation/windows.html

wget https://repo.saltstack.com/windows/Salt-Minion-3003-Py3-AMD64-Setup.exe

Deploy Windows VM via salt:

salt-cloud -p w16k w16k-salty-minion -l debug

Deployed VM you can see firewall and salt minion installed:

Troubleshooting:

[ERROR   ] Unable to execute command
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/salt/utils/cloud.py", line 1005, in wait_for_psexecsvc
    stdout, stderr, ret_code = run_psexec_command(
  File "/usr/lib/python3/dist-packages/salt/utils/cloud.py", line 956, in run_psexec_command
    client = Client(
  File "/usr/lib/python3/dist-packages/salt/utils/cloud.py", line 879, in __init__
    self._client = PsExecClient(server, username, password, port, encrypt)
NameError: name 'PsExecClient' is not defined

cause: PsExecClient module is not installed. , use pip3 to install this

Upgrade LCM

Log in to LCM> Lifecycle operations> Settings> System upgrade

Create snapshot

Once snapshot is done, Click on Check for upgrade,

vRLCM will reboot as a part of the upgrade. Wait patiently… (go have a coffee)

On successfull upgrade, we should see :

Add vRA package repository to LCM

Download vRA 8.3 upgrade repository from https://my.vmware.com/group/vmware/downloads/details?downloadGroup=VRA-830&productId=1116&rPId=59213

copy the ISO to LCM, /data directory

Now, Go back to LCM> Lifecycle operations> settings > Binary mapping>Add product binaries, enter path in base location and then cick on discover

Clock on the prelude(the one we uploaded) and click on add

wait for the import task to compleate. You can take a look at the task by clicking on requests > Click on the most recent request

On successfull import, you should see it in the binary mapping:

Upgrade VRA:

Now that the upgrade packages have been uploaded to LCM, Go into LCM > LifeCycle Operations > Environments > Click on “view details” for the vRA environment.

Triger inventory sync:

Go grab a coffee!!, Typically at this point its just waiting for vRLCM to perform the upgrade.

Click on upgrade:

Run pre-check

wait for pre-check to complete: (takes about 3-10 min)

submit upgrade task

Now, Go grab a cup of coffee, This step generally takes a while.

user: root

when deployed via LCM:
Password: same as appliance password as configured on LCM

When deployed via ovf(vCenter)
password: salt

When attempting to add a product to usage meter, the product migth fail to add if it has a certificate signed by sectigo

Cause: https://support.sectigo.com/articles/Knowledge/Sectigo-AddTrust-External-CA-Root-Expiring-May-30-2020

Resolution: Import root certificates to the appliance java keystone.

Steps:
* take a snapshot of the appliance
* ssh to usage meter appliance
* change user to root

su root

* create or import the root certificate to the appliance

curl https://crt.sh/?d=1199354 >  /home/usagemeter/root.crt

Note: if you have a different CA provider, replace the below with the path to download the root certificate or simply scp the certificate to the UM appliance.

import the certificates (run the command as it is if the root is placed in /home/usagemeter/root.crt

keytool -import -trustcacerts -file /home/usagemeter/root.crt -alias USERTRUST -keystore /usr/java/jre-vmware/lib/security/cacerts

Note: Default keystore password is

changeit

on successfull import, you should see

now, go ahead and add the product back in to usage meter:

Note: when adding vCD, Please ensure that you add the endpoint in the format https://FQDN, IE: https://vcd.ntitta.in

Troubleshooting (show fill certificate chain, check the validity of the last certificate. ):

openssl s_client -showcerts -connect vcsa.ntitta.lab:443