--- /dev/null
+# Ceph Windows Image Build
+
+The Windows image can be generated, fully unattended, via the build script:
+
+```bash
+./build/build
+```
+
+It accepts the following environment variables:
+
+* `SSH_PRIVATE_KEY` (required) - The SSH private key path that will be authorized to access the VMs using the new image.
+* `WINDOWS_SERVER_2019_ISO_URL` (optional) - URL to the Windows Server 2019 ISO image. It defaults to the official Microsoft evaluation ISO.
+* `VIRTIO_WIN_ISO_URL` (optional) - URL to the virtio-win guest tools ISO image. It defaults to the stable ISO from the [official docs](https://github.com/virtio-win/virtio-win-pkg-scripts/blob/master/README.md#downloads).
+
+The build script assumes that the host is a KVM enabled machine, and it will do the following:
+
+1. Download the ISOs from the URLs specified in the environment variables (or the defaults if not given).
+
+2. Start a libvirt virtual machine and install the Windows Server 2019 from the ISO.
+
+ * The process is fully unattended, via the `autounattended.xml` file with the input needed to install the operating system.
+
+ * The virtio drivers and the guest tools are installed from the ISO.
+
+ * SSH is configured and the given SSH private key is authorized.
+
+3. Install the latest Windows updates.
+
+4. Run the `setup.ps1` script to prepare the CI environment.
+
+5. Generalize the VM image via `Sysprep`.
--- /dev/null
+<?xml version="1.0" encoding="utf-8"?>
+<unattend xmlns="urn:schemas-microsoft-com:unattend">
+
+ <settings pass="windowsPE">
+
+ <component name="Microsoft-Windows-International-Core-WinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <SetupUILanguage>
+ <UILanguage>en-US</UILanguage>
+ </SetupUILanguage>
+ <SystemLocale>en-US</SystemLocale>
+ <UILanguage>en-US</UILanguage>
+ <UserLocale>en-US</UserLocale>
+ </component>
+
+ <component name="Microsoft-Windows-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+ <DiskConfiguration>
+ <WillShowUI>OnError</WillShowUI>
+ <Disk wcm:action="add">
+ <CreatePartitions>
+ <CreatePartition wcm:action="add">
+ <Order>1</Order>
+ <Size>100</Size>
+ <Type>Primary</Type>
+ </CreatePartition>
+ <CreatePartition wcm:action="add">
+ <Order>2</Order>
+ <Extend>true</Extend>
+ <Type>Primary</Type>
+ </CreatePartition>
+ </CreatePartitions>
+ <ModifyPartitions>
+ <ModifyPartition wcm:action="add">
+ <Active>true</Active>
+ <Label>Boot</Label>
+ <Format>NTFS</Format>
+ <Order>1</Order>
+ <PartitionID>1</PartitionID>
+ </ModifyPartition>
+ <ModifyPartition wcm:action="add">
+ <Format>NTFS</Format>
+ <Order>2</Order>
+ <PartitionID>2</PartitionID>
+ <Label>System</Label>
+ </ModifyPartition>
+ </ModifyPartitions>
+ <DiskID>0</DiskID>
+ <WillWipeDisk>true</WillWipeDisk>
+ </Disk>
+ </DiskConfiguration>
+
+ <ImageInstall>
+ <OSImage>
+ <InstallTo>
+ <PartitionID>2</PartitionID>
+ <DiskID>0</DiskID>
+ </InstallTo>
+ <InstallToAvailablePartition>false</InstallToAvailablePartition>
+ <WillShowUI>OnError</WillShowUI>
+ <InstallFrom>
+ <MetaData wcm:action="add">
+ <Key>/IMAGE/NAME</Key>
+ <Value>Windows Server 2019 SERVERSTANDARDCORE</Value>
+ </MetaData>
+ </InstallFrom>
+ </OSImage>
+ </ImageInstall>
+
+ <UserData>
+ <!-- Product Key from http://technet.microsoft.com/en-us/library/jj612867.aspx -->
+ <ProductKey>
+ <!-- Do not uncomment the Key element if you are using trial ISOs -->
+ <!-- You must uncomment the Key element (and optionally insert your own key) if you are using retail or volume license ISOs -->
+ <!-- <Key></Key> -->
+ <WillShowUI>OnError</WillShowUI>
+ </ProductKey>
+ <AcceptEula>true</AcceptEula>
+ </UserData>
+
+ </component>
+
+ <component name="Microsoft-Windows-PnpCustomizationsWinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <DriverPaths>
+ <PathAndCredentials wcm:action="add" wcm:keyValue="1">
+ <Path>E:\NetKVM\2k19\amd64\</Path>
+ </PathAndCredentials>
+ <PathAndCredentials wcm:action="add" wcm:keyValue="3">
+ <Path>E:\viostor\2k19\amd64\</Path>
+ </PathAndCredentials>
+ </DriverPaths>
+ </component>
+
+ </settings>
+
+ <settings pass="oobeSystem">
+ <component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+ <VisualEffects>
+ <FontSmoothing>ClearType</FontSmoothing>
+ </VisualEffects>
+
+ <UserAccounts>
+ <!--
+ Password to be used only during initial provisioning.
+ Must be reset with final Sysprep.
+ -->
+ <AdministratorPassword>
+ <Value>Passw0rd</Value>
+ <PlainText>true</PlainText>
+ </AdministratorPassword>
+ </UserAccounts>
+
+ <AutoLogon>
+ <Password>
+ <Value>Passw0rd</Value>
+ <PlainText>true</PlainText>
+ </Password>
+ <Enabled>true</Enabled>
+ <Username>Administrator</Username>
+ </AutoLogon>
+
+ <ComputerName>*</ComputerName>
+
+ <OOBE>
+ <NetworkLocation>Work</NetworkLocation>
+ <HideEULAPage>true</HideEULAPage>
+ <ProtectYourPC>3</ProtectYourPC>
+ <SkipMachineOOBE>true</SkipMachineOOBE>
+ <SkipUserOOBE>true</SkipUserOOBE>
+ </OOBE>
+
+ <FirstLogonCommands>
+
+ <SynchronousCommand wcm:action="add">
+ <CommandLine>%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell -NoLogo -NonInteractive -ExecutionPolicy RemoteSigned -File A:\install-virtio-guest-tools.ps1</CommandLine>
+ <Order>1</Order>
+ </SynchronousCommand>
+
+ <SynchronousCommand wcm:action="add">
+ <CommandLine>%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell -NoLogo -NonInteractive -ExecutionPolicy RemoteSigned -File A:\install-openssh-server.ps1</CommandLine>
+ <Order>2</Order>
+ </SynchronousCommand>
+
+ </FirstLogonCommands>
+
+ </component>
+
+ </settings>
+
+ <settings pass="specialize">
+
+ <component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <TimeZone>UTC</TimeZone>
+ <ComputerName>*</ComputerName>
+ </component>
+
+ </settings>
+
+</unattend>
--- /dev/null
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+
+if [[ -z $SSH_PRIVATE_KEY ]]; then
+ echo "ERROR: The SSH private key secret is not set"
+ exit 1
+fi
+
+WINDOWS_SERVER_2019_ISO_URL=${WINDOWS_SERVER_2019_ISO_URL:-"https://software-download.microsoft.com/download/pr/17763.737.190906-2324.rs5_release_svc_refresh_SERVER_EVAL_x64FRE_en-us_1.iso"}
+VIRTIO_WIN_ISO_URL=${VIRTIO_WIN_ISO_URL:-"https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso"}
+
+BUILD_DIR="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)"
+
+source ${BUILD_DIR}/../../scripts/build_utils.sh
+
+
+function restart_windows_vm() {
+ echo "Restarting Windows VM"
+ ssh_exec "shutdown.exe /r /t 0 & sc.exe stop sshd"
+ SECONDS=0
+ TIMEOUT=${1:-600}
+ while true; do
+ if [[ $SECONDS -gt $TIMEOUT ]]; then
+ echo "Timeout waiting for the VM to start"
+ exit 1
+ fi
+ ssh_exec hostname || {
+ echo "Cannot execute SSH commands yet"
+ sleep 10
+ continue
+ }
+ break
+ done
+ echo "Windows VM restarted"
+}
+
+
+if ! which virt-install >/dev/null; then
+ sudo apt-get update
+ sudo apt-get install -y virtinst
+fi
+
+if ! sudo virsh net-info default &>/dev/null; then
+ cat << EOF > $WORKSPACE/default-net.xml
+<network>
+ <name>default</name>
+ <bridge name="virbr0"/>
+ <forward mode="nat"/>
+ <ip address="192.168.122.1" netmask="255.255.255.0">
+ <dhcp>
+ <range start="192.168.122.2" end="192.168.122.254"/>
+ </dhcp>
+ </ip>
+</network>
+EOF
+ sudo virsh net-define $WORKSPACE/default-net.xml
+ sudo virsh net-start default
+ sudo virsh net-autostart default
+ rm $WORKSPACE/default-net.xml
+fi
+
+echo "Downloading virtio-win ISO"
+retrycmd_if_failure 5 0 30m curl -C - -L $VIRTIO_WIN_ISO_URL -o ${BUILD_DIR}/virtio-win.iso
+
+echo "Downloading Windows Server 2019 ISO"
+retrycmd_if_failure 5 0 60m curl -C - -L $WINDOWS_SERVER_2019_ISO_URL -o ${BUILD_DIR}/windows-server-2019.iso
+
+echo "Creating floppy image"
+qemu-img create -f raw ${BUILD_DIR}/floppy.img 1440k
+mkfs.msdos -s 1 ${BUILD_DIR}/floppy.img
+mkdir ${BUILD_DIR}/floppy
+sudo mount ${BUILD_DIR}/floppy.img ${BUILD_DIR}/floppy
+ssh-keygen -y -f $SSH_PRIVATE_KEY > ${BUILD_DIR}/id_rsa.pub
+sudo cp \
+ ${BUILD_DIR}/autounattend.xml \
+ ${BUILD_DIR}/install-virtio-guest-tools.ps1 \
+ ${BUILD_DIR}/install-openssh-server.ps1 \
+ ${BUILD_DIR}/id_rsa.pub \
+ ${BUILD_DIR}/floppy/
+sudo umount ${BUILD_DIR}/floppy
+rmdir ${BUILD_DIR}/floppy
+
+echo "Starting libvirt VM"
+qemu-img create -f qcow2 ${BUILD_DIR}/ceph-win-ltsc2019-ci-image.qcow2 50G
+VM_NAME="ceph-win-ltsc2019"
+sudo virt-install \
+ --name $VM_NAME \
+ --os-variant win2k19 \
+ --boot hd,cdrom \
+ --virt-type kvm \
+ --graphics spice \
+ --cpu host \
+ --vcpus 4 \
+ --memory 4096 \
+ --disk ${BUILD_DIR}/floppy.img,device=floppy \
+ --disk ${BUILD_DIR}/ceph-win-ltsc2019-ci-image.qcow2,bus=virtio \
+ --disk ${BUILD_DIR}/windows-server-2019.iso,device=cdrom \
+ --disk ${BUILD_DIR}/virtio-win.iso,device=cdrom \
+ --network network=default,model=virtio \
+ --controller type=virtio-serial \
+ --channel unix,target_type=virtio,name=org.qemu.guest_agent.0 \
+ --noautoconsol
+
+export SSH_USER="administrator"
+export SSH_KNOWN_HOSTS_FILE="${BUILD_DIR}/known_hosts"
+export SSH_KEY="$SSH_PRIVATE_KEY"
+
+SECONDS=0
+TIMEOUT=1200
+SLEEP_SECS=30
+while true; do
+ if [[ $SECONDS -gt $TIMEOUT ]]; then
+ echo "Timeout waiting for the VM to start"
+ exit 1
+ fi
+ VM_IP=$(sudo virsh domifaddr --source agent --interface Ethernet --full $VM_NAME | grep ipv4 | awk '{print $4}' | cut -d '/' -f1) || {
+ echo "Retrying in $SLEEP_SECS seconds"
+ sleep $SLEEP_SECS
+ continue
+ }
+ ssh-keyscan -H $VM_IP &> $SSH_KNOWN_HOSTS_FILE || {
+ echo "SSH is not reachable yet"
+ sleep $SLEEP_SECS
+ continue
+ }
+ SSH_ADDRESS=$VM_IP ssh_exec hostname || {
+ echo "Cannot execute SSH commands yet"
+ sleep $SLEEP_SECS
+ continue
+ }
+ break
+done
+export SSH_ADDRESS=$VM_IP
+
+scp_upload ${BUILD_DIR}/install-windows-updates.ps1 /install-windows-updates.ps1
+SSH_TIMEOUT=1h ssh_exec powershell.exe -File /install-windows-updates.ps1
+ssh_exec powershell.exe Remove-Item -Force /install-windows-updates.ps1
+
+restart_windows_vm 1800
+
+scp_upload ${BUILD_DIR}/setup.ps1 /setup.ps1
+SSH_TIMEOUT=1h ssh_exec powershell.exe -File /setup.ps1
+ssh_exec powershell.exe Remove-Item -Force /setup.ps1
+
+restart_windows_vm
+
+sudo virsh qemu-agent-command $VM_NAME \
+ '{"execute":"guest-exec", "arguments":{"path":"ipconfig.exe", "arg":["/release"]}}'
+sudo virsh qemu-agent-command $VM_NAME \
+ '{"execute":"guest-exec", "arguments":{"path":"C:\\Windows\\System32\\Sysprep\\Sysprep.exe", "arg":["/generalize", "/oobe", "/shutdown", "/quiet"]}}'
+
+SECONDS=0
+TIMEOUT=600
+while true; do
+ if [[ $SECONDS -gt $TIMEOUT ]]; then
+ echo "Timeout waiting for the sysprep to shut down the VM"
+ exit 1
+ fi
+ if sudo virsh list | grep -q " $VM_NAME "; then
+ echo "Sysprep is still running"
+ sleep 10
+ continue
+ fi
+ echo "Sysprep finished"
+ break
+done
+
+sudo mv "${BUILD_DIR}/ceph-win-ltsc2019-ci-image.qcow2" ${WORKSPACE}/
+
+${BUILD_DIR}/cleanup
+
+echo "Image successfully built at: ${WORKSPACE}/ceph-win-ltsc2019-ci-image.qcow2"
+
+echo "Uploading the image to https://filedump.ceph.com/windows/"
+cd ${WORKSPACE}
+sha256sum ceph-win-ltsc2019-ci-image.qcow2 > ceph-win-ltsc2019-ci-image.qcow2.sha256
+ssh-keyscan -H drop.front.sepia.ceph.com &>> $SSH_KNOWN_HOSTS_FILE
+rsync -azvP ceph-win-ltsc2019-ci-image.qcow2 ceph-win-ltsc2019-ci-image.qcow2.sha256 --rsync-path="sudo rsync" -e "ssh -i ${FILEDUMP_SSH_KEY} -o UserKnownHostsFile=${SSH_KNOWN_HOSTS_FILE}" ${FILEDUMP_USER}@drop.front.sepia.ceph.com:/ceph/filedump.ceph.com/windows/
--- /dev/null
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+
+BUILD_DIR="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)"
+
+source ${BUILD_DIR}/../../scripts/build_utils.sh
+
+if mountpoint -q -- "${BUILD_DIR}/floppy"; then
+ sudo umount ${BUILD_DIR}/floppy
+fi
+
+delete_libvirt_vms
+clear_libvirt_networks
+
+sudo rm -rf "${BUILD_DIR}/virtio-win.iso" "${BUILD_DIR}/windows-server-2019.iso" \
+ "${BUILD_DIR}/floppy" "${BUILD_DIR}/floppy.img" "${BUILD_DIR}/ceph-win-ltsc2019-ci-image.qcow2" \
+ "${BUILD_DIR}/known_hosts" "${BUILD_DIR}/id_rsa.pub"
--- /dev/null
+$ErrorActionPreference = "Stop"
+
+Get-WindowsCapability -Online -Name OpenSSH* | Add-WindowsCapability -Online
+
+Set-Service -Name "sshd" -StartupType Automatic
+Start-Service -Name "sshd"
+
+New-NetFirewallRule -Name "sshd" -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22
+
+# Authorize the SSH key
+$authorizedKeysFile = Join-Path $env:ProgramData "ssh\administrators_authorized_keys"
+Set-Content -Path $authorizedKeysFile -Value (Get-Content "${PSScriptRoot}\id_rsa.pub") -Encoding ascii
+$acl = Get-Acl $authorizedKeysFile
+$acl.SetAccessRuleProtection($true, $false)
+$administratorsRule = New-Object system.security.accesscontrol.filesystemaccessrule("Administrators", "FullControl", "Allow")
+$systemRule = New-Object system.security.accesscontrol.filesystemaccessrule("SYSTEM", "FullControl", "Allow")
+$acl.SetAccessRule($administratorsRule)
+$acl.SetAccessRule($systemRule)
+$acl | Set-Acl
--- /dev/null
+$ErrorActionPreference = "Stop"
+
+$VIRTIO_WIN_PATH = "E:\"
+
+Write-Output "Installing virtio-win guest tools"
+
+# Trust driver certs
+$certStore = Get-Item "cert:\LocalMachine\TrustedPublisher"
+$certStore.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite)
+$driverPath = Get-Item "${VIRTIO_WIN_PATH}\*\2k19\amd64"
+Get-ChildItem -Recurse -Path $driverPath -Filter "*.cat" | ForEach-Object {
+ $cert = (Get-AuthenticodeSignature $_.FullName).SignerCertificate
+ $certStore.Add($cert)
+}
+$certStore.Close()
+
+# Install QEMU quest tools
+$p = Start-Process -FilePath "${VIRTIO_WIN_PATH}\virtio-win-guest-tools.exe" -ArgumentList @("/install", "/quiet", "/norestart") -NoNewWindow -PassThru -Wait
+if($p.ExitCode) {
+ Throw "The virtio-win guest tools installation failed. Exit code: $($p.ExitCode)"
+}
+
+Write-Output "Successfully installed virtio-win guest tools"
--- /dev/null
+$ErrorActionPreference = "Stop"
+$ProgressPreference='SilentlyContinue'
+
+Write-Output "Installing PSWindowsUpdate PowerShell module"
+Install-PackageProvider -Name "NuGet" -Force -Confirm:$false
+Set-PSRepository -Name "PSGallery" -InstallationPolicy Trusted
+Install-Module -Name "PSWindowsUpdate" -Force -Confirm:$false
+
+Write-Output "Installing latest Windows updates"
+$updateScript = {
+ Import-Module "PSWindowsUpdate"
+ Install-WindowsUpdate -AcceptAll -IgnoreReboot | Out-File -FilePath "${env:SystemDrive}\PSWindowsUpdate.log" -Encoding ascii
+}
+Invoke-WUJob -Script $updateScript -Confirm:$false -RunNow
+while($true) {
+ $task = Get-ScheduledTask -TaskName "PSWindowsUpdate"
+ if($task.State -eq "Ready") {
+ break
+ }
+ Start-Sleep -Seconds 10
+}
+Get-Content "${env:SystemDrive}\PSWindowsUpdate.log"
+Remove-Item -Force -Path "${env:SystemDrive}\PSWindowsUpdate.log"
+Unregister-ScheduledTask -TaskName "PSWindowsUpdate" -Confirm:$false
+Write-Output "Windows updates successfully installed"
--- /dev/null
+$ErrorActionPreference = "Stop"
+$ProgressPreference = "SilentlyContinue"
+
+$VS_2019_BUILD_TOOLS_URL = "https://aka.ms/vs/16/release/vs_buildtools.exe"
+$WDK_URL = "https://download.microsoft.com/download/7/d/6/7d602355-8ae9-414c-ae36-109ece2aade6/wdk/wdksetup.exe" # Windows 11 WDK (22000.1). It can be used to develop drivers for previous OS releases.
+$PYTHON3_URL = "https://www.python.org/ftp/python/3.10.1/python-3.10.1-amd64.exe"
+
+$WNBD_GIT_REPO = "https://github.com/ceph/wnbd.git"
+$WNBD_GIT_BRANCH = "master"
+
+
+function Get-WindowsBuildInfo {
+ $p = Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion"
+ $table = New-Object System.Data.DataTable
+ $table.Columns.AddRange(@("Release", "Version", "Build"))
+ $table.Rows.Add($p.ProductName, $p.ReleaseId, "$($p.CurrentBuild).$($p.UBR)") | Out-Null
+ return $table
+}
+
+function Invoke-CommandLine {
+ Param(
+ [Parameter(Mandatory=$true)]
+ [String]$Command,
+ [String]$Arguments,
+ [Int[]]$AllowedExitCodes=@(0)
+ )
+ & $Command $Arguments.Split(" ")
+ if($LASTEXITCODE -notin $AllowedExitCodes) {
+ Throw "$Command $Arguments returned a non zero exit code ${LASTEXITCODE}."
+ }
+}
+
+function Start-ExecuteWithRetry {
+ Param(
+ [Parameter(Mandatory=$true)]
+ [ScriptBlock]$ScriptBlock,
+ [Int]$MaxRetryCount=10,
+ [Int]$RetryInterval=3,
+ [String]$RetryMessage,
+ [Array]$ArgumentList=@()
+ )
+ $currentErrorActionPreference = $ErrorActionPreference
+ $ErrorActionPreference = "Continue"
+ $retryCount = 0
+ while ($true) {
+ try {
+ $res = Invoke-Command -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList
+ $ErrorActionPreference = $currentErrorActionPreference
+ return $res
+ } catch [System.Exception] {
+ $retryCount++
+ if ($retryCount -gt $MaxRetryCount) {
+ $ErrorActionPreference = $currentErrorActionPreference
+ Throw $_
+ } else {
+ $prefixMsg = "Retry(${retryCount}/${MaxRetryCount})"
+ if($RetryMessage) {
+ Write-Host "${prefixMsg} - $RetryMessage"
+ } elseif($_) {
+ Write-Host "${prefixMsg} - $($_.ToString())"
+ }
+ Start-Sleep $RetryInterval
+ }
+ }
+ }
+}
+
+function Start-FileDownload {
+ Param(
+ [Parameter(Mandatory=$true)]
+ [String]$URL,
+ [Parameter(Mandatory=$true)]
+ [String]$Destination,
+ [Int]$RetryCount=10
+ )
+ Write-Output "Downloading $URL to $Destination"
+ Start-ExecuteWithRetry `
+ -ScriptBlock { Invoke-CommandLine -Command "curl.exe" -Arguments "-L -s -o $Destination $URL" } `
+ -MaxRetryCount $RetryCount `
+ -RetryMessage "Failed to download '${URL}'. Retrying"
+ Write-Output "Successfully downloaded."
+}
+
+function Add-ToPathEnvVar {
+ Param(
+ [Parameter(Mandatory=$true)]
+ [String[]]$Path,
+ [Parameter(Mandatory=$false)]
+ [ValidateSet([System.EnvironmentVariableTarget]::User, [System.EnvironmentVariableTarget]::Machine)]
+ [System.EnvironmentVariableTarget]$Target=[System.EnvironmentVariableTarget]::Machine
+ )
+ $pathEnvVar = [Environment]::GetEnvironmentVariable("PATH", $Target).Split(';')
+ $currentSessionPath = $env:PATH.Split(';')
+ foreach($p in $Path) {
+ if($p -notin $pathEnvVar) {
+ $pathEnvVar += $p
+ }
+ if($p -notin $currentSessionPath) {
+ $currentSessionPath += $p
+ }
+ }
+ $env:PATH = $currentSessionPath -join ';'
+ $newPathEnvVar = $pathEnvVar -join ';'
+ [Environment]::SetEnvironmentVariable("PATH", $newPathEnvVar, $Target)
+}
+
+function Install-Tool {
+ [CmdletBinding(DefaultParameterSetName = "URL")]
+ Param(
+ [Parameter(Mandatory=$true, ParameterSetName = "URL")]
+ [String]$URL,
+ [Parameter(Mandatory=$true, ParameterSetName = "LocalPath")]
+ [String]$LocalPath,
+ [Parameter(ParameterSetName = "URL")]
+ [Parameter(ParameterSetName = "LocalPath")]
+ [String[]]$Params=@(),
+ [Parameter(ParameterSetName = "URL")]
+ [Parameter(ParameterSetName = "LocalPath")]
+ [Int[]]$AllowedExitCodes=@(0)
+ )
+ PROCESS {
+ $installerPath = $LocalPath
+ if($PSCmdlet.ParameterSetName -eq "URL") {
+ $installerPath = Join-Path $env:TEMP $URL.Split('/')[-1]
+ Start-FileDownload -URL $URL -Destination $installerPath
+ }
+ Write-Output "Installing ${installerPath}"
+ $p = Start-Process -FilePath $installerPath -ArgumentList $Params -NoNewWindow -PassThru -Wait
+ if($p.ExitCode -notin $AllowedExitCodes) {
+ Throw "Installation failed. Exit code: $($p.ExitCode)"
+ }
+ if($PSCmdlet.ParameterSetName -eq "URL") {
+ Start-ExecuteWithRetry `
+ -ScriptBlock { Remove-Item -Force -Path $installerPath -ErrorAction Stop } `
+ -RetryMessage "Failed to remove ${installerPath}. Retrying"
+ }
+ }
+}
+
+function Get-GitHubReleaseAssets {
+ Param(
+ [Parameter(Mandatory=$true)]
+ [String]$Repository,
+ [String]$Version="latest"
+ )
+ $releasesUrl = "https://api.github.com/repos/${Repository}/releases"
+ if($Version -eq "latest") {
+ $release = Invoke-CommandLine "curl.exe" "-s ${releasesUrl}/latest" | ConvertFrom-Json
+ } else {
+ $releases = Invoke-CommandLine "curl.exe" "-s ${releasesUrl}" | ConvertFrom-Json
+ $release = $releases | Where-Object { $_.tag_name -eq $Version }
+ if(!$release) {
+ Throw "Cannot find '${Repository}' release '${Version}'."
+ }
+ }
+ return $release.assets
+}
+
+function Set-VCVars {
+ Param(
+ [String]$Version="2019",
+ [String]$Platform="x86_amd64"
+ )
+ Push-Location "${env:ProgramFiles(x86)}\Microsoft Visual Studio\${Version}\BuildTools\VC\Auxiliary\Build"
+ try {
+ cmd.exe /c "vcvarsall.bat ${Platform} & set" | ForEach-Object {
+ if ($_ -match "=") {
+ $v = $_.split("=")
+ Set-Item -Force -Path "ENV:\$($v[0])" -Value "$($v[1])"
+ }
+ }
+ }
+ finally {
+ Pop-Location
+ }
+}
+
+function Install-Requirements {
+ # Create the needed directories
+ New-Item `
+ -ItemType Directory -Force `
+ -Path @(
+ "${env:SystemDrive}\tmp",
+ "${env:SystemDrive}\ceph",
+ "${env:SystemDrive}\wnbd",
+ "${env:SystemDrive}\workspace",
+ "${env:SystemDrive}\workspace\repos"
+ )
+ Add-ToPathEnvVar -Path @("${env:SystemDrive}\ceph", "${env:SystemDrive}\wnbd")
+ # Set UTC time zone
+ Set-TimeZone -Id "UTC"
+ # Allow ping requests
+ Get-NetFirewallRule -Name @("FPS-ICMP4-ERQ-In", "FPS-ICMP6-ERQ-In") | Enable-NetFirewallRule
+ # Allow test signed drivers
+ Invoke-CommandLine "bcdedit.exe" "/set testsigning yes"
+}
+
+function Install-VisualStudio2019BuildTools {
+ Write-Output "Installing Visual Studio 2019 Build Tools"
+ $params = @(
+ "--quiet", "--wait", "--norestart", "--nocache",
+ "--add", "Microsoft.VisualStudio.Workload.VCTools",
+ "--add", "Microsoft.VisualStudio.Workload.MSBuildTools",
+ "--add", "Microsoft.VisualStudio.Component.VC.Runtimes.x86.x64.Spectre",
+ "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+ "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22000"
+ )
+ Install-Tool -URL $VS_2019_BUILD_TOOLS_URL -Params $params -AllowedExitCodes @(0, 3010)
+ Write-Output "Successfully installed Visual Studio 2019 Build Tools"
+}
+
+function Install-WDK {
+ # Install WDK excluding WDK.vsix
+ Write-Output "Installing Windows Development Kit (WDK)"
+ Install-Tool -URL $WDK_URL -Params @("/q")
+ # Install WDK.vsix in manual manner
+ Copy-Item -Path "${env:ProgramFiles(x86)}\Windows Kits\10\Vsix\VS2019\WDK.vsix" -Destination "${env:TEMP}\wdkvsix.zip"
+ Expand-Archive "${env:TEMP}\wdkvsix.zip" -DestinationPath "${env:TEMP}\wdkvsix"
+ $src = "${env:TEMP}\wdkvsix\`$MSBuild\Microsoft\VC\v160"
+ $dst = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2019\BuildTools\MSBuild\Microsoft\VC\v160"
+ New-Item -ItemType Directory -Force -Path $dst | Out-Null
+ Push-Location $src
+ Get-ChildItem -Recurse | Resolve-Path -Relative | ForEach-Object {
+ $item = Get-Item -Path $_
+ if($item.PSIsContainer) {
+ New-Item -ItemType Directory -Force -Path "${dst}\$_" | Out-Null
+ } else {
+ Copy-Item -Force -Path $item.FullName -Destination "${dst}\$_" | Out-Null
+ }
+ }
+ Pop-Location
+ Remove-Item -Recurse -Force -Path @("${env:TEMP}\wdkvsix.zip", "${env:TEMP}\wdkvsix")
+ Write-Output "Successfully installed Windows Development Kit (WDK)"
+}
+
+function Get-GitDownloadUrl {
+ Param(
+ [String]$Version="latest"
+ )
+ $asset = Get-GitHubReleaseAssets -Repository "git-for-windows/git" -Version $Version | Where-Object {
+ $_.content_type -eq "application/executable" -and `
+ $_.name.StartsWith("Git-") -and `
+ $_.name.EndsWith("-64-bit.exe") }
+ if(!$asset) {
+ Throw "Cannot find Git on Windows release asset with the 64-bit installer."
+ }
+ if($asset.Count -gt 1) {
+ Throw "Found multiple Git on Windows release assets with the 64-bit installer."
+ }
+ return $asset.browser_download_url
+}
+
+function Install-Git {
+ Write-Output "Installing Git"
+ Install-Tool -URL (Get-GitDownloadUrl) -Params @("/VERYSILENT", "/NORESTART")
+ Add-ToPathEnvVar -Path @("${env:ProgramFiles}\Git\cmd", "${env:ProgramFiles}\Git\usr\bin")
+ Write-Output "Successfully installed Git"
+}
+
+function Install-WnbdDriver {
+ $outDir = Join-Path $env:SystemDrive "wnbd"
+ $gitDir = Join-Path $env:TEMP "wnbd"
+ Invoke-CommandLine "git.exe" "clone ${WNBD_GIT_REPO} --branch ${WNBD_GIT_BRANCH} ${gitDir}"
+ Push-Location $gitDir
+ try {
+ Set-VCVars
+ Invoke-CommandLine "MSBuild.exe" "vstudio\wnbd.sln /p:Configuration=Release"
+ Copy-Item -Force -Path "vstudio\x64\Release\driver\*" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\x64\Release\libwnbd.dll" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\x64\Release\wnbd-client.exe" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\x64\Release\wnbd.cer" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\x64\Release\pdb\driver\*" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\x64\Release\pdb\libwnbd\*" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\x64\Release\pdb\wnbd-client\*" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\wnbdevents.xml" -Destination "${outDir}\"
+ Copy-Item -Force -Path "vstudio\reinstall.ps1" -Destination "${outDir}\"
+ } finally {
+ Pop-Location
+ Remove-Item -Recurse -Force -Path $gitDir
+ }
+ Import-Certificate -FilePath "${outDir}\wnbd.cer" -Cert Cert:\LocalMachine\Root
+ Import-Certificate -FilePath "${outDir}\wnbd.cer" -Cert Cert:\LocalMachine\TrustedPublisher
+ Invoke-CommandLine "wnbd-client.exe" "install-driver ${outDir}\wnbd.inf"
+ Invoke-CommandLine "wevtutil.exe" "im ${outDir}\wnbdevents.xml"
+}
+
+function Install-Python3 {
+ Write-Output "Installing Python3"
+ Install-Tool -URL $PYTHON3_URL -Params @("/quiet", "InstallAllUsers=1", "PrependPath=1")
+ Add-ToPathEnvVar -Path @("${env:ProgramFiles}\Python310\", "${env:ProgramFiles}\Python310\Scripts\")
+ Write-Output "Installing pip dependencies"
+ Start-ExecuteWithRetry {
+ # Needed to run the Ceph unit tests via https://github.com/ceph/ceph-win32-tests scripts
+ Invoke-CommandLine "pip3.exe" "install os-testr python-dateutil requests"
+ }
+ Write-Output "Successfully installed Python3"
+}
+
+function Get-Wix3ToolsetDownloadUrl {
+ Param(
+ [String]$Version="latest"
+ )
+ $asset = Get-GitHubReleaseAssets -Repository "wixtoolset/wix3" -Version $Version | Where-Object {
+ $_.content_type -eq "application/x-msdownload" }
+ if(!$asset) {
+ Throw "Cannot find Wix3 toolset release asset."
+ }
+ if($asset.Count -gt 1) {
+ Throw "Found multiple Wix3 toolset release assets."
+ }
+ return $asset.browser_download_url
+}
+
+function Install-Wix3Toolset {
+ Write-Output "Installing .NET Framework 3.5"
+ Install-WindowsFeature -Name "NET-Framework-Features" -ErrorAction Stop
+ Write-Output "Installing Wix3 toolset"
+ Install-Tool -URL (Get-Wix3ToolsetDownloadUrl) -Params @("/install", "/quiet", "/norestart")
+ Write-Output "Successfully installed Wix3 toolset"
+}
+
+
+Get-WindowsBuildInfo
+Install-Requirements
+Install-VisualStudio2019BuildTools
+Install-WDK
+Install-Git
+Install-WnbdDriver
+Install-Python3
+Install-Wix3Toolset
+
+Write-Output "Successfully installed the CI environment. Please reboot the system before sysprep."
--- /dev/null
+- job:
+ name: ceph-windows-image-build
+ description: 'Builds the Ceph Windows VM image used in the CI.'
+ node: amd64 && focal && libvirt
+ project-type: freestyle
+ defaults: global
+ concurrent: false
+ display-name: 'ceph-windows-image-build'
+ properties:
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ artifact-days-to-keep: 30
+ artifact-num-to-keep: 30
+
+ parameters:
+ - string:
+ name: WINDOWS_SERVER_2019_ISO_URL
+ description: "The Windows Server 2019 ISO URL."
+ default: https://software-download.microsoft.com/download/pr/17763.737.190906-2324.rs5_release_svc_refresh_SERVER_EVAL_x64FRE_en-us_1.iso
+
+ - string:
+ name: VIRTIO_WIN_ISO_URL
+ description: "The virtio-win guest tools ISO URL."
+ default: https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso
+
+ scm:
+ - git:
+ url: https://github.com/ceph/ceph-build.git
+ branches:
+ - master
+ basedir: ceph-build
+
+ builders:
+ - shell: "${WORKSPACE}/ceph-build/ceph-windows-image-build/build/build"
+
+ wrappers:
+ - credentials-binding:
+ - file:
+ credential-id: ceph_win_ci_private_key
+ variable: SSH_PRIVATE_KEY
+ - ssh-user-private-key:
+ credential-id: CEPH_WINDOWS_FILEDUMP_SSH_KEY
+ key-file-variable: FILEDUMP_SSH_KEY
+ username-variable: FILEDUMP_USER
+
+ publishers:
+ - postbuildscript:
+ builders:
+ - role: SLAVE
+ build-on:
+ - UNSTABLE
+ - FAILURE
+ - ABORTED
+ build-steps:
+ - shell: "${WORKSPACE}/ceph-build/ceph-windows-image-build/build/cleanup"
fi
timeout ${SSH_TIMEOUT:-"30s"} scp -i ${SSH_KEY:-"$HOME/.ssh/id_rsa"} $SSH_OPTS -r ${SSH_USER}@${SSH_ADDRESS}:${REMOTE_FILE} $LOCAL_FILE
}
+
+function retrycmd_if_failure() {
+ set +o errexit
+ retries=$1
+ wait_sleep=$2
+ timeout=$3
+ shift && shift && shift
+ for i in $(seq 1 "$retries"); do
+ if timeout "$timeout" "${@}"; then
+ break
+ fi
+ if [[ $i -eq $retries ]]; then
+ echo "Error: Failed to execute '$*' after $i attempts"
+ set -o errexit
+ return 1
+ fi
+ sleep "$wait_sleep"
+ done
+ echo "Executed '$*' $i times"
+ set -o errexit
+}
projects / ceph-build.git / commitdiff