From 5bceefb06310ac223f8d023be5619c883f19291e Mon Sep 17 00:00:00 2001 From: Nathan Gieseker Date: Thu, 7 Nov 2019 20:05:19 -0800 Subject: [PATCH 01/33] add functions for getting vm state --- vm.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/vm.go b/vm.go index ced94af3f1..7b8ce42eb8 100644 --- a/vm.go +++ b/vm.go @@ -5,6 +5,7 @@ import ( "encoding/json" "github.com/Microsoft/hcsshim/internal/hcs" hcsschema "github.com/Microsoft/hcsshim/internal/schema2" + "github.com/Microsoft/hcsshim/internal/schema1" "time" ) @@ -92,6 +93,24 @@ func getHcsSpec(system *hcs.System) *hcsschema.ComputeSystem { return nil } +func GetVirtualMachineState(id string) string { + properties, err := GetVirtualMachineProperties(id) + if err != nil { + return "" + } + return properties.State +} + +func GetVirtualMachineProperties(id string) (*schema1.ContainerProperties, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + system, err := hcs.OpenComputeSystem(ctx, id) + if err != nil { + return nil, err + } + return system.Properties(ctx) +} + func GetVirtualMachineSpec(id string) (*VirtualMachineSpec, error) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() From 502ee5620d958aa31dbec0bb3b0af14993b3106f Mon Sep 17 00:00:00 2001 From: Nathan Gieseker Date: Wed, 4 Dec 2019 17:46:46 -0800 Subject: [PATCH 02/33] Add CommonParams to get the status for hcn objects --- hcn/hcn.go | 20 ++++++++++++++++++++ hcn/hcnendpoint.go | 1 + hcn/hcnloadbalancer.go | 1 + hcn/hcnnetwork.go | 1 + 4 files changed, 23 insertions(+) diff --git a/hcn/hcn.go b/hcn/hcn.go index da741449b8..dc2865b1b8 100644 --- a/hcn/hcn.go +++ b/hcn/hcn.go @@ -175,3 +175,23 @@ var ( // RequestTypeRefresh refreshes the settings provided. RequestTypeRefresh RequestType = "Refresh" ) + +type ResourcesParams struct { + state int `json:"State,omitempty"` +} + +type ExtraParams struct { + resources ResourcesParams `json:"Resources,omitempty"` +} + +type HealthParams struct { + extra ExtraParams `json:"Extra,omitempty"` +} + +type CommonParams struct { + health HealthParams `json:"Health,omitempty"` +} + +func (c CommonParams) GetState() int { + return c.health.extra.resources.state +} diff --git a/hcn/hcnendpoint.go b/hcn/hcnendpoint.go index 829de929f2..d13db56454 100644 --- a/hcn/hcnendpoint.go +++ b/hcn/hcnendpoint.go @@ -27,6 +27,7 @@ var ( // HostComputeEndpoint represents a network endpoint type HostComputeEndpoint struct { + CommonParams Id string `json:"ID,omitempty"` Name string `json:",omitempty"` HostComputeNetwork string `json:",omitempty"` // GUID diff --git a/hcn/hcnloadbalancer.go b/hcn/hcnloadbalancer.go index 898e02a801..d9208aa7d8 100644 --- a/hcn/hcnloadbalancer.go +++ b/hcn/hcnloadbalancer.go @@ -18,6 +18,7 @@ type LoadBalancerPortMapping struct { // HostComputeLoadBalancer represents software load balancer. type HostComputeLoadBalancer struct { + CommonParams Id string `json:"ID,omitempty"` HostComputeEndpoints []string `json:",omitempty"` SourceVIP string `json:",omitempty"` diff --git a/hcn/hcnnetwork.go b/hcn/hcnnetwork.go index a95fc1d751..be19c9df9a 100644 --- a/hcn/hcnnetwork.go +++ b/hcn/hcnnetwork.go @@ -74,6 +74,7 @@ const ( // HostComputeNetwork represents a network type HostComputeNetwork struct { + CommonParams Id string `json:"ID,omitempty"` Name string `json:",omitempty"` Type NetworkType `json:",omitempty"` From e6631fb3afb52d4d25c270dc42a82176ee6dab3d Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 11 Mar 2021 21:22:09 -0800 Subject: [PATCH 03/33] Add Hot Attach/Detach Endpoints to VirtualMachineSpec --- vm.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/vm.go b/vm.go index 7b8ce42eb8..4f1f958a0f 100644 --- a/vm.go +++ b/vm.go @@ -3,10 +3,14 @@ package hcsshim import ( "context" "encoding/json" + "path" + "time" + + "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/internal/hcs" - hcsschema "github.com/Microsoft/hcsshim/internal/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" "github.com/Microsoft/hcsshim/internal/schema1" - "time" + hcsschema "github.com/Microsoft/hcsshim/internal/schema2" ) type VirtualMachineSpec struct { @@ -110,7 +114,7 @@ func GetVirtualMachineProperties(id string) (*schema1.ContainerProperties, error } return system.Properties(ctx) } - + func GetVirtualMachineSpec(id string) (*VirtualMachineSpec, error) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() @@ -250,6 +254,67 @@ func (vm *VirtualMachineSpec) ExecuteCommand(command string) error { return nil } +func (vm *VirtualMachineSpec) HotAttachEndpoints(endpoints []*hcn.HostComputeEndpoint) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + for _, endpoint := range endpoints { + if err = vm.hotAttachEndpoint(ctx, system, endpoint); err != nil { + return err + } + } + return nil +} + +func (vm *VirtualMachineSpec) HotDetachEndpoint(endpoint *hcn.HostComputeEndpoint) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + // Hot detach an endpoint from the compute system + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + ResourcePath: path.Join("VirtualMachine/Devices/NetworkAdapters", endpoint.Id), + Settings: hcsschema.NetworkAdapter{ + EndpointId: endpoint.Id, + MacAddress: endpoint.MacAddress, + }, + } + + if err = system.Modify(ctx, request); err != nil { + return err + } + + return nil +} + +func (vm *VirtualMachineSpec) hotAttachEndpoint(ctx context.Context, system *hcs.System, endpoint *hcn.HostComputeEndpoint) (err error) { + // Hot attach an endpoint to the compute system + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + ResourcePath: path.Join("VirtualMachine/Devices/NetworkAdapters", endpoint.Id), + Settings: hcsschema.NetworkAdapter{ + EndpointId: endpoint.Id, + MacAddress: endpoint.MacAddress, + }, + } + + if err = system.Modify(ctx, request); err != nil { + return err + } + + return nil +} + func (vm *VirtualMachineSpec) String() string { jsonString, err := json.Marshal(vm.spec) if err != nil { From f3b1ed7a17947ad0412132e69ba067876c3115f7 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Mon, 15 Mar 2021 15:24:50 -0700 Subject: [PATCH 04/33] Add Wait() to compute system --- vm.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vm.go b/vm.go index 4f1f958a0f..f41e3e7892 100644 --- a/vm.go +++ b/vm.go @@ -112,6 +112,8 @@ func GetVirtualMachineProperties(id string) (*schema1.ContainerProperties, error if err != nil { return nil, err } + defer system.Close() + return system.Properties(ctx) } @@ -241,6 +243,19 @@ func (vm *VirtualMachineSpec) Delete() error { return system.Terminate(ctx) } +// Wait for a Virtual Machine exits +func (vm *VirtualMachineSpec) Wait() error { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + return system.Wait() +} + // ExecuteCommand executes a command in the Virtual Machine func (vm *VirtualMachineSpec) ExecuteCommand(command string) error { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) From 190507ad580a151e391d78b29618ba681c361b34 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Tue, 13 Apr 2021 10:35:15 -0700 Subject: [PATCH 05/33] Revert "Add CommonParams to get the status for hcn objects" This reverts commit 502ee5620d958aa31dbec0bb3b0af14993b3106f. --- hcn/hcn.go | 20 -------------------- hcn/hcnendpoint.go | 1 - hcn/hcnloadbalancer.go | 1 - hcn/hcnnetwork.go | 1 - 4 files changed, 23 deletions(-) diff --git a/hcn/hcn.go b/hcn/hcn.go index dc2865b1b8..da741449b8 100644 --- a/hcn/hcn.go +++ b/hcn/hcn.go @@ -175,23 +175,3 @@ var ( // RequestTypeRefresh refreshes the settings provided. RequestTypeRefresh RequestType = "Refresh" ) - -type ResourcesParams struct { - state int `json:"State,omitempty"` -} - -type ExtraParams struct { - resources ResourcesParams `json:"Resources,omitempty"` -} - -type HealthParams struct { - extra ExtraParams `json:"Extra,omitempty"` -} - -type CommonParams struct { - health HealthParams `json:"Health,omitempty"` -} - -func (c CommonParams) GetState() int { - return c.health.extra.resources.state -} diff --git a/hcn/hcnendpoint.go b/hcn/hcnendpoint.go index d13db56454..829de929f2 100644 --- a/hcn/hcnendpoint.go +++ b/hcn/hcnendpoint.go @@ -27,7 +27,6 @@ var ( // HostComputeEndpoint represents a network endpoint type HostComputeEndpoint struct { - CommonParams Id string `json:"ID,omitempty"` Name string `json:",omitempty"` HostComputeNetwork string `json:",omitempty"` // GUID diff --git a/hcn/hcnloadbalancer.go b/hcn/hcnloadbalancer.go index d9208aa7d8..898e02a801 100644 --- a/hcn/hcnloadbalancer.go +++ b/hcn/hcnloadbalancer.go @@ -18,7 +18,6 @@ type LoadBalancerPortMapping struct { // HostComputeLoadBalancer represents software load balancer. type HostComputeLoadBalancer struct { - CommonParams Id string `json:"ID,omitempty"` HostComputeEndpoints []string `json:",omitempty"` SourceVIP string `json:",omitempty"` diff --git a/hcn/hcnnetwork.go b/hcn/hcnnetwork.go index be19c9df9a..a95fc1d751 100644 --- a/hcn/hcnnetwork.go +++ b/hcn/hcnnetwork.go @@ -74,7 +74,6 @@ const ( // HostComputeNetwork represents a network type HostComputeNetwork struct { - CommonParams Id string `json:"ID,omitempty"` Name string `json:",omitempty"` Type NetworkType `json:",omitempty"` From 67c329dac06f10dd0795ae8d3eae2a2be9735969 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 15 Apr 2021 15:05:00 -0700 Subject: [PATCH 06/33] Correct HcnRegisterServiceCallback function prototype --- hcn/hcn.go | 2 +- hcn/zsyscall_windows.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hcn/hcn.go b/hcn/hcn.go index da741449b8..b210c21514 100644 --- a/hcn/hcn.go +++ b/hcn/hcn.go @@ -57,7 +57,7 @@ import ( // Service //sys hcnOpenService(service *hcnService, result **uint16) (hr error) = computenetwork.HcnOpenService? -//sys hcnRegisterServiceCallback(service hcnService, callback int32, context int32, callbackHandle *hcnCallbackHandle) (hr error) = computenetwork.HcnRegisterServiceCallback? +//sys hcnRegisterServiceCallback(callback uintptr, context uintptr, callbackHandle *hcnCallbackHandle) (hr error) = computenetwork.HcnRegisterServiceCallback? //sys hcnUnregisterServiceCallback(callbackHandle hcnCallbackHandle) (hr error) = computenetwork.HcnUnregisterServiceCallback? //sys hcnCloseService(service hcnService) (hr error) = computenetwork.HcnCloseService? diff --git a/hcn/zsyscall_windows.go b/hcn/zsyscall_windows.go index 856b2c1408..5852faa02c 100644 --- a/hcn/zsyscall_windows.go +++ b/hcn/zsyscall_windows.go @@ -671,11 +671,11 @@ func hcnOpenService(service *hcnService, result **uint16) (hr error) { return } -func hcnRegisterServiceCallback(service hcnService, callback int32, context int32, callbackHandle *hcnCallbackHandle) (hr error) { +func hcnRegisterServiceCallback(callback uintptr, context uintptr, callbackHandle *hcnCallbackHandle) (hr error) { if hr = procHcnRegisterServiceCallback.Find(); hr != nil { return } - r0, _, _ := syscall.Syscall6(procHcnRegisterServiceCallback.Addr(), 4, uintptr(service), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0) + r0, _, _ := syscall.Syscall(procHcnRegisterServiceCallback.Addr(), 3, uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle))) if int32(r0) < 0 { if r0&0x1fff0000 == 0x00070000 { r0 &= 0xffff From db417e29f5396eda8ff215f2e776e39fbd8d477d Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 15 Apr 2021 15:08:21 -0700 Subject: [PATCH 07/33] Add HcnRegister/UnregisterNetworkCallback --- hcn/hcn.go | 2 ++ hcn/zsyscall_windows.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/hcn/hcn.go b/hcn/hcn.go index b210c21514..7da8c860ce 100644 --- a/hcn/hcn.go +++ b/hcn/hcn.go @@ -27,6 +27,8 @@ import ( //sys hcnQueryNetworkProperties(network hcnNetwork, query string, properties **uint16, result **uint16) (hr error) = computenetwork.HcnQueryNetworkProperties? //sys hcnDeleteNetwork(id *_guid, result **uint16) (hr error) = computenetwork.HcnDeleteNetwork? //sys hcnCloseNetwork(network hcnNetwork) (hr error) = computenetwork.HcnCloseNetwork? +//sys hcsRegisterNetworkCallback(network hcnNetwork, callback uintptr, context uintptr, callbackHandle *hcnCallbackHandle) (hr error) = computenetwork.HcnRegisterNetworkCallback? +//sys hcsUnregisterNetworkCallback(callbackHandle hcnCallbackHandle) (hr error) = computenetwork.HcnUnregisterNetworkCallback? // Endpoint //sys hcnEnumerateEndpoints(query string, endpoints **uint16, result **uint16) (hr error) = computenetwork.HcnEnumerateEndpoints? diff --git a/hcn/zsyscall_windows.go b/hcn/zsyscall_windows.go index 5852faa02c..b323dc2f74 100644 --- a/hcn/zsyscall_windows.go +++ b/hcn/zsyscall_windows.go @@ -75,6 +75,8 @@ var ( procHcnRegisterServiceCallback = modcomputenetwork.NewProc("HcnRegisterServiceCallback") procHcnUnregisterServiceCallback = modcomputenetwork.NewProc("HcnUnregisterServiceCallback") procHcnCloseService = modcomputenetwork.NewProc("HcnCloseService") + procHcnRegisterNetworkCallback = modcomputenetwork.NewProc("HcnRegisterNetworkCallback") + procHcnUnregisterNetworkCallback = modcomputenetwork.NewProc("HcnUnregisterNetworkCallback") ) func SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) { @@ -712,3 +714,31 @@ func hcnCloseService(service hcnService) (hr error) { } return } + +func hcnRegisterNetworkCallback(network hcnNetwork, callback uintptr, context uintptr, callbackHandle *hcnCallbackHandle) (hr error) { + if hr = procHcnRegisterNetworkCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnRegisterNetworkCallback.Addr(), 4, uintptr(network), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnUnregisterNetworkCallback(callbackHandle hcnCallbackHandle) (hr error) { + if hr = procHcnUnregisterNetworkCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnUnregisterNetworkCallback.Addr(), 1, uintptr(callbackHandle), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} From cd5981c7865b4dac9e19dc2442055c75f29119d6 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 15 Apr 2021 16:29:39 -0700 Subject: [PATCH 08/33] Implement HCN Service Watcher --- hcn/hcncallback.go | 121 +++++++++++++++++++++++++++++++++++++++++++++ hcn/hcnnetwork.go | 104 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 hcn/hcncallback.go diff --git a/hcn/hcncallback.go b/hcn/hcncallback.go new file mode 100644 index 0000000000..87cc169d3d --- /dev/null +++ b/hcn/hcncallback.go @@ -0,0 +1,121 @@ +package hcn + +import ( + "encoding/json" + "fmt" + "sync" + "syscall" + "unsafe" + + "github.com/Microsoft/hcsshim/internal/interop" +) + +var ( + nextCallback uintptr + callbackMap = map[uintptr]*notifcationWatcherContext{} + callbackMapLock = sync.RWMutex{} + + notificationWatcherCallback = syscall.NewCallback(notificationWatcher) + + // Notifications for HCN_SERVICE handles + HcnNotificationNetworkPreCreate HcnNotification = 0x00000001 + HcnNotificationNetworkCreate HcnNotification = 0x00000002 + HcnNotificationNetworkPreDelete HcnNotification = 0x00000003 + HcnNotificationNetworkDelete HcnNotification = 0x00000004 + + // Namespace Notifications + HcnNotificationNamespaceCreate HcnNotification = 0x00000005 + HcnNotificationNamespaceDelete HcnNotification = 0x00000006 + + // Notifications for HCN_SERVICE handles + HcnNotificationGuestNetworkServiceCreate HcnNotification = 0x00000007 + HcnNotificationGuestNetworkServiceDelete HcnNotification = 0x00000008 + + // Notifications for HCN_NETWORK handles + HcnNotificationNetworkEndpointAttached HcnNotification = 0x00000009 + HcnNotificationNetworkEndpointDetached HcnNotification = 0x00000010 + + // Notifications for HCN_GUESTNETWORKSERVICE handles + HcnNotificationGuestNetworkServiceStateChanged HcnNotification = 0x00000011 + HcnNotificationGuestNetworkServiceInterfaceStateChanged HcnNotification = 0x00000012 + + // Common notifications + HcnNotificationServiceDisconnect HcnNotification = 0x01000000 +) + +type HcnNotification uint32 + +func (hn HcnNotification) String() string { + switch hn { + case HcnNotificationNetworkPreCreate: + return "NetworkPreCreate" + case HcnNotificationNetworkCreate: + return "NetworkCreate" + case HcnNotificationNetworkPreDelete: + return "NetworkPreDelete" + case HcnNotificationNetworkDelete: + return "NetworkDelete" + case HcnNotificationNamespaceCreate: + return "NamespaceCreate" + case HcnNotificationNamespaceDelete: + return "NamespaceDelete" + case HcnNotificationGuestNetworkServiceCreate: + return "GuestNetworkServiceCreate" + case HcnNotificationGuestNetworkServiceDelete: + return "GuestNetworkServiceDelete" + case HcnNotificationNetworkEndpointAttached: + return "NetworkEndpointAttached" + case HcnNotificationNetworkEndpointDetached: + return "NetworkEndpointDetached" + case HcnNotificationGuestNetworkServiceStateChanged: + return "GuestNetworkServiceStateChanged" + case HcnNotificationGuestNetworkServiceInterfaceStateChanged: + return "GuestNetworkServiceInterfaceStateChanged" + case HcnNotificationServiceDisconnect: + return "ServiceDisconnect" + default: + return fmt.Sprintf("Unknown: %d", hn) + } +} + +type HcnNotificationData struct { + Type HcnNotification + Status error + Data NotificationBase +} + +type notificationChannel chan HcnNotificationData + +type notifcationWatcherContext struct { + channel notificationChannel + handle hcnCallbackHandle +} + +func notificationWatcher(notificationType HcnNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr { + var result error + if int32(notificationStatus) < 0 { + result = interop.Win32FromHresult(notificationStatus) + } + + callbackMapLock.RLock() + context := callbackMap[callbackNumber] + callbackMapLock.RUnlock() + + if context == nil { + return 0 + } + + tmpString := syscall.UTF16ToString((*[1 << 29]uint16)(unsafe.Pointer(notificationData))[:]) + + var notifData NotificationBase + err := json.Unmarshal([]byte(tmpString), ¬ifData) + if err == nil { + context.channel <- HcnNotificationData{ + Type: notificationType, + Status: result, + Data: notifData, + } + } + + return 0 +} diff --git a/hcn/hcnnetwork.go b/hcn/hcnnetwork.go index a95fc1d751..b85cbafc1c 100644 --- a/hcn/hcnnetwork.go +++ b/hcn/hcnnetwork.go @@ -3,6 +3,7 @@ package hcn import ( "encoding/json" "errors" + "sync" "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/interop" @@ -109,6 +110,109 @@ type PolicyNetworkRequest struct { Policies []NetworkPolicy `json:",omitempty"` } +type NotificationBase struct { + Id guid.GUID `json:"ID"` + Flags uint32 `json:",omitempty"` + Data json.RawMessage `json:",omitempty"` +} + +type HcnServiceWatcher struct { + handleLock sync.RWMutex + callbackNumber uintptr + watcherContext *notifcationWatcherContext + started bool +} + +func NewHcnServiceWatcher() *HcnServiceWatcher { + watcherContext := ¬ifcationWatcherContext{ + channel: make(notificationChannel, 1), + } + + callbackMapLock.Lock() + callbackNumber := nextCallback + nextCallback++ + callbackMap[callbackNumber] = watcherContext + callbackMapLock.Unlock() + + return &HcnServiceWatcher{ + callbackNumber: callbackNumber, + watcherContext: watcherContext, + } +} + +func (w *HcnServiceWatcher) Start() error { + w.handleLock.RLock() + defer w.handleLock.RUnlock() + + if w.started { + return nil + } + + var callbackHandle hcnCallbackHandle + err := hcnRegisterServiceCallback(notificationWatcherCallback, w.callbackNumber, &callbackHandle) + if err != nil { + return err + } + + w.watcherContext.handle = callbackHandle + w.started = true + return nil +} + +func (w *HcnServiceWatcher) Stop() error { + var handle hcnCallbackHandle + + { + w.handleLock.RLock() + defer w.handleLock.RUnlock() + + if !w.started { + return nil + } + w.started = false + handle = w.watcherContext.handle + w.watcherContext.handle = 0 + } + + if handle == 0 { + return nil + } + + // hcnUnregisterServiceCallback has its own syncronization + // to wait for all callbacks to complete. We must NOT hold the callbackMapLock. + err := hcnUnregisterServiceCallback(handle) + if err != nil { + return err + } + + return nil +} + +func (w *HcnServiceWatcher) Notification() <-chan HcnNotificationData { + w.handleLock.RLock() + defer w.handleLock.RUnlock() + + if !w.started { + return nil + } + return w.watcherContext.channel +} + +func (w *HcnServiceWatcher) Close() error { + w.Stop() + + w.handleLock.RLock() + close(w.watcherContext.channel) + callbackNumber := w.callbackNumber + w.handleLock.RUnlock() + + callbackMapLock.Lock() + delete(callbackMap, callbackNumber) + callbackMapLock.Unlock() + + return nil +} + func getNetwork(networkGuid guid.GUID, query string) (*HostComputeNetwork, error) { // Open network. var ( From 285188498cf8d0a4c93c9a61458456ab0f98736c Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Mon, 19 Apr 2021 09:43:46 -0700 Subject: [PATCH 09/33] add option for enabling GuestConnection --- vm.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/vm.go b/vm.go index f41e3e7892..ebdc0d5fc9 100644 --- a/vm.go +++ b/vm.go @@ -21,7 +21,7 @@ type VirtualMachineSpec struct { system *hcs.System } -func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryInMB, processorCount int, vnicId, macAddress string) (*VirtualMachineSpec, error) { +func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryInMB, processorCount int, vnicId, macAddress string, guestconnection bool) (*VirtualMachineSpec, error) { spec := &hcsschema.ComputeSystem{ Owner: owner, SchemaVersion: &hcsschema.Version{ @@ -64,10 +64,6 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn }, NetworkAdapters: map[string]hcsschema.NetworkAdapter{}, }, - // GuestConnection: &hcsschema.GuestConnection{ - // UseVsock: true, - // UseConnectedSuspend: true, - //}, }, } @@ -78,6 +74,13 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn } } + if guestconnection { + spec.VirtualMachine.GuestConnection = &hcsschema.GuestConnection{ + UseVsock: true, + UseConnectedSuspend: true, + } + } + return &VirtualMachineSpec{ spec: spec, ID: id, From 1335addf653e848155103ab5a3e28b1527939dcc Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Tue, 20 Apr 2021 09:29:03 -0700 Subject: [PATCH 10/33] Use VirtualMachineOptions to create vm spec --- vm.go | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/vm.go b/vm.go index ebdc0d5fc9..afc2f32a5a 100644 --- a/vm.go +++ b/vm.go @@ -13,6 +13,19 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/schema2" ) +type VirtualMachineOptions struct { + Name string + Id string + VhdPath string + IsoPath string + Owner string + MemoryInMB int32 + ProcessorCount int32 + VnicId string + MacAddress string + UseGuestConnection bool +} + type VirtualMachineSpec struct { Name string ID string @@ -21,9 +34,9 @@ type VirtualMachineSpec struct { system *hcs.System } -func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryInMB, processorCount int, vnicId, macAddress string, guestconnection bool) (*VirtualMachineSpec, error) { +func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, error) { spec := &hcsschema.ComputeSystem{ - Owner: owner, + Owner: opts.Owner, SchemaVersion: &hcsschema.Version{ Major: 2, Minor: 1, @@ -41,10 +54,10 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn }, ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.Memory2{ - SizeInMB: int32(memoryInMB), + SizeInMB: int32(opts.MemoryInMB), }, Processor: &hcsschema.Processor2{ - Count: int32(processorCount), + Count: int32(opts.ProcessorCount), }, }, Devices: &hcsschema.Devices{ @@ -52,11 +65,11 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn "primary": hcsschema.Scsi{ Attachments: map[string]hcsschema.Attachment{ "0": hcsschema.Attachment{ - Path: vhdPath, + Path: opts.VhdPath, Type_: "VirtualDisk", }, "1": hcsschema.Attachment{ - Path: isoPath, + Path: opts.IsoPath, Type_: "Iso", }, }, @@ -67,14 +80,14 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn }, } - if len(vnicId) > 0 { + if len(opts.VnicId) > 0 { spec.VirtualMachine.Devices.NetworkAdapters["ext"] = hcsschema.NetworkAdapter{ - EndpointId: vnicId, - MacAddress: macAddress, + EndpointId: opts.VnicId, + MacAddress: opts.MacAddress, } } - if guestconnection { + if opts.UseGuestConnection { spec.VirtualMachine.GuestConnection = &hcsschema.GuestConnection{ UseVsock: true, UseConnectedSuspend: true, @@ -83,8 +96,8 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn return &VirtualMachineSpec{ spec: spec, - ID: id, - Name: name, + ID: opts.Id, + Name: opts.Name, }, nil } From cb9d84a952997df56ffda6851054968b1bc03130 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Tue, 20 Apr 2021 11:22:44 -0700 Subject: [PATCH 11/33] Add support for plan9 file sharing --- vm.go | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/vm.go b/vm.go index afc2f32a5a..91994f398f 100644 --- a/vm.go +++ b/vm.go @@ -3,14 +3,18 @@ package hcsshim import ( "context" "encoding/json" + "errors" + "fmt" "path" "time" "github.com/Microsoft/hcsshim/hcn" + "github.com/Microsoft/hcsshim/internal/guestrequest" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/requesttype" "github.com/Microsoft/hcsshim/internal/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/schema2" + "github.com/Microsoft/hcsshim/osversion" ) type VirtualMachineOptions struct { @@ -26,6 +30,8 @@ type VirtualMachineOptions struct { UseGuestConnection bool } +const plan9Port = 564 + type VirtualMachineSpec struct { Name string ID string @@ -346,6 +352,108 @@ func (vm *VirtualMachineSpec) hotAttachEndpoint(ctx context.Context, system *hcs return nil } +// AddPlan9 adds a Plan9 share to a VirtualMachineSpec. +func (vm *VirtualMachineSpec) AddPlan9(shareName string, hostPath string, uvmPath string, readOnly bool, restrict bool, allowedNames []string) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + if restrict && osversion.Get().Build < 18328 { + return errors.New("single-file mappings are not supported on this build of Windows") + } + if uvmPath == "" { + return fmt.Errorf("uvmPath must be passed to AddPlan9") + } + + // TODO: JTERRY75 - These are marked private in the schema. For now use them + // but when there are public variants we need to switch to them. + const ( + shareFlagsReadOnly int32 = 0x00000001 + shareFlagsLinuxMetadata int32 = 0x00000004 + shareFlagsCaseSensitive int32 = 0x00000008 + shareFlagsRestrictFileAccess int32 = 0x00000080 + ) + + // TODO: JTERRY75 - `shareFlagsCaseSensitive` only works if the Windows + // `hostPath` supports case sensitivity. We need to detect this case before + // forwarding this flag in all cases. + flags := shareFlagsLinuxMetadata // | shareFlagsCaseSensitive + if readOnly { + flags |= shareFlagsReadOnly + } + if restrict { + flags |= shareFlagsRestrictFileAccess + } + + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.Plan9Share{ + Name: shareName, + AccessName: shareName, + Path: hostPath, + Port: plan9Port, + Flags: flags, + AllowedFiles: allowedNames, + }, + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/Plan9/Shares"), + GuestRequest: guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeMappedDirectory, + RequestType: requesttype.Add, + Settings: guestrequest.LCOWMappedDirectory{ + MountPath: uvmPath, + ShareName: shareName, + Port: plan9Port, + ReadOnly: readOnly, + }, + }, + } + + if err := system.Modify(ctx, modification); err != nil { + return err + } + + return nil +} + +func (vm *VirtualMachineSpec) RemovePlan9(shareName string, uvmPath string) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + Settings: hcsschema.Plan9Share{ + Name: shareName, + AccessName: shareName, + Port: plan9Port, + }, + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/Plan9/Shares"), + GuestRequest: guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeMappedDirectory, + RequestType: requesttype.Remove, + Settings: guestrequest.LCOWMappedDirectory{ + MountPath: uvmPath, + ShareName: shareName, + Port: plan9Port, + }, + }, + } + if err := system.Modify(ctx, modification); err != nil { + return fmt.Errorf("failed to remove plan9 share %s from %s: %+v: %s", shareName, vm.ID, modification, err) + } + return nil +} + func (vm *VirtualMachineSpec) String() string { jsonString, err := json.Marshal(vm.spec) if err != nil { From 87bdee70e0be0c46bbcb80a31143798afbcb8043 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Tue, 20 Apr 2021 11:22:44 -0700 Subject: [PATCH 12/33] Add support for plan9 file sharing --- vm.go | 1 + 1 file changed, 1 insertion(+) diff --git a/vm.go b/vm.go index 91994f398f..ab96231cc7 100644 --- a/vm.go +++ b/vm.go @@ -82,6 +82,7 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, }, }, NetworkAdapters: map[string]hcsschema.NetworkAdapter{}, + Plan9: &hcsschema.Plan9{}, }, }, } From c592abcb66b682f52125239df26c81f0d032cf29 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Mon, 19 Apr 2021 09:43:46 -0700 Subject: [PATCH 13/33] Use VirtualMachineOptions to create vm spec --- vm.go | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/vm.go b/vm.go index f41e3e7892..afc2f32a5a 100644 --- a/vm.go +++ b/vm.go @@ -13,6 +13,19 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/schema2" ) +type VirtualMachineOptions struct { + Name string + Id string + VhdPath string + IsoPath string + Owner string + MemoryInMB int32 + ProcessorCount int32 + VnicId string + MacAddress string + UseGuestConnection bool +} + type VirtualMachineSpec struct { Name string ID string @@ -21,9 +34,9 @@ type VirtualMachineSpec struct { system *hcs.System } -func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryInMB, processorCount int, vnicId, macAddress string) (*VirtualMachineSpec, error) { +func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, error) { spec := &hcsschema.ComputeSystem{ - Owner: owner, + Owner: opts.Owner, SchemaVersion: &hcsschema.Version{ Major: 2, Minor: 1, @@ -41,10 +54,10 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn }, ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.Memory2{ - SizeInMB: int32(memoryInMB), + SizeInMB: int32(opts.MemoryInMB), }, Processor: &hcsschema.Processor2{ - Count: int32(processorCount), + Count: int32(opts.ProcessorCount), }, }, Devices: &hcsschema.Devices{ @@ -52,11 +65,11 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn "primary": hcsschema.Scsi{ Attachments: map[string]hcsschema.Attachment{ "0": hcsschema.Attachment{ - Path: vhdPath, + Path: opts.VhdPath, Type_: "VirtualDisk", }, "1": hcsschema.Attachment{ - Path: isoPath, + Path: opts.IsoPath, Type_: "Iso", }, }, @@ -64,24 +77,27 @@ func CreateVirtualMachineSpec(name, id, vhdPath, isoPath, owner string, memoryIn }, NetworkAdapters: map[string]hcsschema.NetworkAdapter{}, }, - // GuestConnection: &hcsschema.GuestConnection{ - // UseVsock: true, - // UseConnectedSuspend: true, - //}, }, } - if len(vnicId) > 0 { + if len(opts.VnicId) > 0 { spec.VirtualMachine.Devices.NetworkAdapters["ext"] = hcsschema.NetworkAdapter{ - EndpointId: vnicId, - MacAddress: macAddress, + EndpointId: opts.VnicId, + MacAddress: opts.MacAddress, + } + } + + if opts.UseGuestConnection { + spec.VirtualMachine.GuestConnection = &hcsschema.GuestConnection{ + UseVsock: true, + UseConnectedSuspend: true, } } return &VirtualMachineSpec{ spec: spec, - ID: id, - Name: name, + ID: opts.Id, + Name: opts.Name, }, nil } From face9d3651a431f6f69bcc9b72faf33aafd540cd Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Tue, 20 Apr 2021 11:22:44 -0700 Subject: [PATCH 14/33] Add support for plan9 file sharing --- vm.go | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/vm.go b/vm.go index afc2f32a5a..ab96231cc7 100644 --- a/vm.go +++ b/vm.go @@ -3,14 +3,18 @@ package hcsshim import ( "context" "encoding/json" + "errors" + "fmt" "path" "time" "github.com/Microsoft/hcsshim/hcn" + "github.com/Microsoft/hcsshim/internal/guestrequest" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/requesttype" "github.com/Microsoft/hcsshim/internal/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/schema2" + "github.com/Microsoft/hcsshim/osversion" ) type VirtualMachineOptions struct { @@ -26,6 +30,8 @@ type VirtualMachineOptions struct { UseGuestConnection bool } +const plan9Port = 564 + type VirtualMachineSpec struct { Name string ID string @@ -76,6 +82,7 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, }, }, NetworkAdapters: map[string]hcsschema.NetworkAdapter{}, + Plan9: &hcsschema.Plan9{}, }, }, } @@ -346,6 +353,108 @@ func (vm *VirtualMachineSpec) hotAttachEndpoint(ctx context.Context, system *hcs return nil } +// AddPlan9 adds a Plan9 share to a VirtualMachineSpec. +func (vm *VirtualMachineSpec) AddPlan9(shareName string, hostPath string, uvmPath string, readOnly bool, restrict bool, allowedNames []string) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + if restrict && osversion.Get().Build < 18328 { + return errors.New("single-file mappings are not supported on this build of Windows") + } + if uvmPath == "" { + return fmt.Errorf("uvmPath must be passed to AddPlan9") + } + + // TODO: JTERRY75 - These are marked private in the schema. For now use them + // but when there are public variants we need to switch to them. + const ( + shareFlagsReadOnly int32 = 0x00000001 + shareFlagsLinuxMetadata int32 = 0x00000004 + shareFlagsCaseSensitive int32 = 0x00000008 + shareFlagsRestrictFileAccess int32 = 0x00000080 + ) + + // TODO: JTERRY75 - `shareFlagsCaseSensitive` only works if the Windows + // `hostPath` supports case sensitivity. We need to detect this case before + // forwarding this flag in all cases. + flags := shareFlagsLinuxMetadata // | shareFlagsCaseSensitive + if readOnly { + flags |= shareFlagsReadOnly + } + if restrict { + flags |= shareFlagsRestrictFileAccess + } + + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.Plan9Share{ + Name: shareName, + AccessName: shareName, + Path: hostPath, + Port: plan9Port, + Flags: flags, + AllowedFiles: allowedNames, + }, + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/Plan9/Shares"), + GuestRequest: guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeMappedDirectory, + RequestType: requesttype.Add, + Settings: guestrequest.LCOWMappedDirectory{ + MountPath: uvmPath, + ShareName: shareName, + Port: plan9Port, + ReadOnly: readOnly, + }, + }, + } + + if err := system.Modify(ctx, modification); err != nil { + return err + } + + return nil +} + +func (vm *VirtualMachineSpec) RemovePlan9(shareName string, uvmPath string) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + Settings: hcsschema.Plan9Share{ + Name: shareName, + AccessName: shareName, + Port: plan9Port, + }, + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/Plan9/Shares"), + GuestRequest: guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeMappedDirectory, + RequestType: requesttype.Remove, + Settings: guestrequest.LCOWMappedDirectory{ + MountPath: uvmPath, + ShareName: shareName, + Port: plan9Port, + }, + }, + } + if err := system.Modify(ctx, modification); err != nil { + return fmt.Errorf("failed to remove plan9 share %s from %s: %+v: %s", shareName, vm.ID, modification, err) + } + return nil +} + func (vm *VirtualMachineSpec) String() string { jsonString, err := json.Marshal(vm.spec) if err != nil { From d13cef57d7614a35d2f4b8dda1368d37a420bebd Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Wed, 21 Apr 2021 16:08:37 -0700 Subject: [PATCH 15/33] Add UpdateGpuConfiguration for GPU-PV --- internal/requesttype/types.go | 1 + internal/schema2/gpu_configuration.go | 24 ++++++++++++++++ vm.go | 41 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 internal/schema2/gpu_configuration.go diff --git a/internal/requesttype/types.go b/internal/requesttype/types.go index 8c0e1b4555..df2db709b0 100644 --- a/internal/requesttype/types.go +++ b/internal/requesttype/types.go @@ -7,4 +7,5 @@ const ( Add = "Add" Remove = "Remove" PreAdd = "PreAdd" // For networking + Update = "Update" ) diff --git a/internal/schema2/gpu_configuration.go b/internal/schema2/gpu_configuration.go new file mode 100644 index 0000000000..f71792a498 --- /dev/null +++ b/internal/schema2/gpu_configuration.go @@ -0,0 +1,24 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type GpuConfiguration struct { + + // The mode used to assign GPUs to the guest. + AssignmentMode string `json:"AssignmentMode,omitempty"` + + // This only applies to List mode, and is ignored in other modes. + // In GPU-P, string is GPU device interface, and unit16 is partition id. HCS simply assigns the partition with the input id. + // In GPU-PV, string is GPU device interface, and unit16 is 0xffff. HCS needs to find an available partition to assign. + AssignmentRequest map[string]uint16 `json:"AssignmentRequest,omitempty"` + + // Whether we allow vendor extension. + AllowVendorExtension bool `json:"AllowVendorExtension,omitempty"` +} diff --git a/vm.go b/vm.go index ab96231cc7..82778a8306 100644 --- a/vm.go +++ b/vm.go @@ -17,6 +17,15 @@ import ( "github.com/Microsoft/hcsshim/osversion" ) +type GpuAssignmentMode string + +const ( + GpuAssignmentModeDisabled = GpuAssignmentMode("Disabled") + GpuAssignmentModeDefault = GpuAssignmentMode("Default") + GpuAssignmentModeList = GpuAssignmentMode("List") + GpuAssignmentModeMirror = GpuAssignmentMode("Mirror") +) + type VirtualMachineOptions struct { Name string Id string @@ -455,6 +464,38 @@ func (vm *VirtualMachineSpec) RemovePlan9(shareName string, uvmPath string) (err return nil } +func (vm *VirtualMachineSpec) UpdateGpuConfiguration(mode GpuAssignmentMode, allowVendorExtension bool, assignments map[string]uint16) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + settings := hcsschema.GpuConfiguration{ + AssignmentMode: string(mode), + AllowVendorExtension: allowVendorExtension, + } + + if len(assignments) != 0 { + settings.AssignmentRequest = assignments + } + + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Update, + ResourcePath: "VirtualMachine/ComputeTopology/Gpu", + Settings: settings, + } + + if err := system.Modify(ctx, request); err != nil { + return err + } + + return nil +} + func (vm *VirtualMachineSpec) String() string { jsonString, err := json.Marshal(vm.spec) if err != nil { From 42e78ce6ae0402aba13dbe9131be8462bac5d83e Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 29 Apr 2021 09:41:07 -0700 Subject: [PATCH 16/33] Grant Vm access to vhd and iso files --- vm.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vm.go b/vm.go index 82778a8306..70b213defa 100644 --- a/vm.go +++ b/vm.go @@ -14,6 +14,7 @@ import ( "github.com/Microsoft/hcsshim/internal/requesttype" "github.com/Microsoft/hcsshim/internal/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/schema2" + "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" ) @@ -50,6 +51,14 @@ type VirtualMachineSpec struct { } func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, error) { + // Ensure the VM has access, we use opts.Id to create VM + if err := wclayer.GrantVmAccess(opts.Id, opts.VhdPath); err != nil { + return nil, err + } + if err := wclayer.GrantVmAccess(opts.Id, opts.IsoPath); err != nil { + return nil, err + } + spec := &hcsschema.ComputeSystem{ Owner: opts.Owner, SchemaVersion: &hcsschema.Version{ From 553268e0ed811cd561eac9f43050d3346d09b26c Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 29 Apr 2021 14:40:37 -0700 Subject: [PATCH 17/33] Remove redundant type --- vm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm.go b/vm.go index 70b213defa..a1cd0f96d5 100644 --- a/vm.go +++ b/vm.go @@ -86,13 +86,13 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, }, Devices: &hcsschema.Devices{ Scsi: map[string]hcsschema.Scsi{ - "primary": hcsschema.Scsi{ + "primary": { Attachments: map[string]hcsschema.Attachment{ - "0": hcsschema.Attachment{ + "0": { Path: opts.VhdPath, Type_: "VirtualDisk", }, - "1": hcsschema.Attachment{ + "1": { Path: opts.IsoPath, Type_: "Iso", }, From 1bcce1f6d3c3da7bb2161b1f912252c1c86054ed Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 29 Apr 2021 16:02:34 -0700 Subject: [PATCH 18/33] Add ShutdownOptions support --- container.go | 2 +- internal/hcs/system.go | 4 +-- internal/schema2/shutdown_options.go | 25 +++++++++++++++++ vm.go | 41 ++++++++++++++++++++++++++-- 4 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 internal/schema2/shutdown_options.go diff --git a/container.go b/container.go index 53c0a3854a..bc60e0a857 100644 --- a/container.go +++ b/container.go @@ -104,7 +104,7 @@ func (container *container) Start() error { // Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds. func (container *container) Shutdown() error { - err := container.system.Shutdown(context.Background()) + err := container.system.Shutdown(context.Background(), "") if err != nil { return convertSystemError(err, container) } diff --git a/internal/hcs/system.go b/internal/hcs/system.go index e1cb78a6fb..274b43f5a3 100644 --- a/internal/hcs/system.go +++ b/internal/hcs/system.go @@ -316,7 +316,7 @@ func (computeSystem *System) ID() string { } // Shutdown requests a compute system shutdown. -func (computeSystem *System) Shutdown(ctx gcontext.Context) (err error) { +func (computeSystem *System) Shutdown(ctx gcontext.Context, options string) (err error) { computeSystem.handleLock.RLock() defer computeSystem.handleLock.RUnlock() @@ -331,7 +331,7 @@ func (computeSystem *System) Shutdown(ctx gcontext.Context) (err error) { } var resultp *uint16 - err = hcsShutdownComputeSystemContext(ctx, computeSystem.handle, "", &resultp) + err = hcsShutdownComputeSystemContext(ctx, computeSystem.handle, options, &resultp) events := processHcsResult(ctx, resultp) switch err { case nil, ErrVmcomputeAlreadyStopped, ErrComputeSystemDoesNotExist, ErrVmcomputeOperationPending: diff --git a/internal/schema2/shutdown_options.go b/internal/schema2/shutdown_options.go new file mode 100644 index 0000000000..7ab7c7bea8 --- /dev/null +++ b/internal/schema2/shutdown_options.go @@ -0,0 +1,25 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ShutdownOptions struct { + + // What kind of mechanism used to perform the shutdown operation + Mechanism string `json:"Mechanism,omitempty"` + + // What is the type of the shutdown operation + Type string `json:"Type,omitempty"` + + // If this shutdown is forceful or not + Force bool `json:"Force,omitempty"` + + // Reason for the shutdown + Reason string `json:"Reason,omitempty"` +} diff --git a/vm.go b/vm.go index a1cd0f96d5..186a6939d8 100644 --- a/vm.go +++ b/vm.go @@ -27,6 +27,21 @@ const ( GpuAssignmentModeMirror = GpuAssignmentMode("Mirror") ) +type ShutdownMechanism string + +const ( + ShutdownMechanismGuestConnection = ShutdownMechanism("GuestConnection") + ShutdownMechanismIntegrationService = ShutdownMechanism("IntegrationService") +) + +type ShutdownType string + +const ( + ShutdownTypeShutdown = ShutdownType("Shutdown") + ShutdownTypeHibernate = ShutdownType("Hibernate") + ShutdownTypeReboot = ShutdownType("Reboot") +) + type VirtualMachineOptions struct { Name string Id string @@ -255,7 +270,7 @@ func (vm *VirtualMachineSpec) Start() error { } // Stop a Virtual Machine -func (vm *VirtualMachineSpec) Stop() error { +func (vm *VirtualMachineSpec) Stop(force bool) error { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() system, err := hcs.OpenComputeSystem(ctx, vm.ID) @@ -264,7 +279,12 @@ func (vm *VirtualMachineSpec) Stop() error { } defer system.Close() - return system.Shutdown(ctx) + shutdownOptions, err := generateShutdownOptions(force) + if err != nil { + return err + } + + return system.Shutdown(ctx, shutdownOptions) } // Delete a Virtual Machine @@ -513,3 +533,20 @@ func (vm *VirtualMachineSpec) String() string { return string(jsonString) } + +func generateShutdownOptions(force bool) (string, error) { + // TODO: shutdown options only supported at schema version 2.5 and above + // check current schema version on the running system and return empty string if + // running on schema version 2.4 and below + options := hcsschema.ShutdownOptions{ + Mechanism: string(ShutdownMechanismGuestConnection), + Type: string(ShutdownTypeShutdown), + Force: force, + Reason: "Requested shutdown", + } + optionsB, err := json.Marshal(options) + if err != nil { + return "", err + } + return string(optionsB), nil +} From e4d23442c77dd11731ebb83cc9e48ee0cb8e023a Mon Sep 17 00:00:00 2001 From: Jagadish Murugan Date: Fri, 28 May 2021 12:28:10 -0700 Subject: [PATCH 19/33] Add API to Add/Remove virtual device (#7) * Add API to Add/Remove virtual device * fix review comment --- internal/guestrequest/types.go | 5 ++ internal/schema2/devices.go | 4 ++ internal/schema2/virtual_pci_device.go | 16 ++++++ internal/schema2/virtual_pci_function.go | 18 +++++++ vm.go | 69 +++++++++++++++++++++++- 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 internal/schema2/virtual_pci_device.go create mode 100644 internal/schema2/virtual_pci_function.go diff --git a/internal/guestrequest/types.go b/internal/guestrequest/types.go index 3dafbdc00a..8ce139cc18 100644 --- a/internal/guestrequest/types.go +++ b/internal/guestrequest/types.go @@ -49,6 +49,10 @@ type LCOWMappedVPMemDevice struct { MountPath string `json:"MountPath,omitempty"` // /tmp/pN } +type LCOWMappedVPCIDevice struct { + VMBusGUID string `json:"VMBusGUID,omitempty"` +} + type LCOWNetworkAdapter struct { NamespaceID string `json:",omitempty"` ID string `json:",omitempty"` @@ -72,6 +76,7 @@ const ( ResourceTypeNetworkNamespace ResourceType = "NetworkNamespace" ResourceTypeCombinedLayers ResourceType = "CombinedLayers" ResourceTypeVPMemDevice ResourceType = "VPMemDevice" + ResourceTypeVPCIDevice ResourceType = "VPCIDevice" ) // GuestRequest is for modify commands passed to the guest. diff --git a/internal/schema2/devices.go b/internal/schema2/devices.go index 781a884015..e985d96d22 100644 --- a/internal/schema2/devices.go +++ b/internal/schema2/devices.go @@ -39,4 +39,8 @@ type Devices struct { FlexibleIov map[string]FlexibleIoDevice `json:"FlexibleIov,omitempty"` SharedMemory *SharedMemoryConfiguration `json:"SharedMemory,omitempty"` + + // TODO: This is pre-release support in schema 2.3. Need to add build number + // docs when a public build with this is out. + VirtualPci map[string]VirtualPciDevice `json:",omitempty"` } diff --git a/internal/schema2/virtual_pci_device.go b/internal/schema2/virtual_pci_device.go new file mode 100644 index 0000000000..f5e05903c5 --- /dev/null +++ b/internal/schema2/virtual_pci_device.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.3 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// TODO: This is pre-release support in schema 2.3. Need to add build number +// docs when a public build with this is out. +type VirtualPciDevice struct { + Functions []VirtualPciFunction `json:",omitempty"` +} diff --git a/internal/schema2/virtual_pci_function.go b/internal/schema2/virtual_pci_function.go new file mode 100644 index 0000000000..cedb7d18bc --- /dev/null +++ b/internal/schema2/virtual_pci_function.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.3 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// TODO: This is pre-release support in schema 2.3. Need to add build number +// docs when a public build with this is out. +type VirtualPciFunction struct { + DeviceInstancePath string `json:",omitempty"` + + VirtualFunction uint16 `json:",omitempty"` +} diff --git a/vm.go b/vm.go index 186a6939d8..367334bc5e 100644 --- a/vm.go +++ b/vm.go @@ -8,6 +8,7 @@ import ( "path" "time" + "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/internal/guestrequest" "github.com/Microsoft/hcsshim/internal/hcs" @@ -53,6 +54,7 @@ type VirtualMachineOptions struct { VnicId string MacAddress string UseGuestConnection bool + AllowOvercommit bool } const plan9Port = 564 @@ -93,7 +95,8 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, }, ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.Memory2{ - SizeInMB: int32(opts.MemoryInMB), + SizeInMB: int32(opts.MemoryInMB), + AllowOvercommit: opts.AllowOvercommit, }, Processor: &hcsschema.Processor2{ Count: int32(opts.ProcessorCount), @@ -525,6 +528,70 @@ func (vm *VirtualMachineSpec) UpdateGpuConfiguration(mode GpuAssignmentMode, all return nil } +// Add vPCI device +func (vm *VirtualMachineSpec) AssignDevice(ctx context.Context, deviceID string) (string, error) { + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return "", err + } + defer system.Close() + + guid, err := guid.NewV4() + if err != nil { + return "", err + } + + vmBusGUID := guid.String() + targetDevice := hcsschema.VirtualPciDevice{ + Functions: []hcsschema.VirtualPciFunction{ + { + DeviceInstancePath: deviceID, + }, + }, + } + request := &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPci/%s", vmBusGUID), + RequestType: requesttype.Add, + Settings: targetDevice, + } + + // for LCOW, we need to make sure that specific paths relating to the + // device exist so they are ready to be used by later + // work in openGCS + request.GuestRequest = guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeVPCIDevice, + RequestType: requesttype.Add, + Settings: guestrequest.LCOWMappedVPCIDevice{ + VMBusGUID: vmBusGUID, + }, + } + + if err := system.Modify(ctx, request); err != nil { + return "", err + } + + return vmBusGUID, nil +} + +// Removes a vpci device from VirtualMachineSpec +func (vm *VirtualMachineSpec) RemoveDevice(ctx context.Context, vmBusGUID string) error { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + + defer system.Close() + + return system.Modify(ctx, &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPci/%s", vmBusGUID), + RequestType: requesttype.Remove, + }) + return nil +} + func (vm *VirtualMachineSpec) String() string { jsonString, err := json.Marshal(vm.spec) if err != nil { From cf4c60cc2b47541c48ba52314f1d58098af239a0 Mon Sep 17 00:00:00 2001 From: Johnson Shih Date: Fri, 11 Jun 2021 14:11:14 -0700 Subject: [PATCH 20/33] skip if notificationData is null (#8) --- hcn/hcncallback.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hcn/hcncallback.go b/hcn/hcncallback.go index 87cc169d3d..d7f1fd193e 100644 --- a/hcn/hcncallback.go +++ b/hcn/hcncallback.go @@ -105,6 +105,10 @@ func notificationWatcher(notificationType HcnNotification, callbackNumber uintpt return 0 } + if notificationData == nil { + return 0 + } + tmpString := syscall.UTF16ToString((*[1 << 29]uint16)(unsafe.Pointer(notificationData))[:]) var notifData NotificationBase From de8040616b0c483b03f13327658abbd0d47701b3 Mon Sep 17 00:00:00 2001 From: Johnson Shih Date: Thu, 22 Jul 2021 17:47:31 -0700 Subject: [PATCH 21/33] correct logic for EndpointNotFoundError (#9) --- hcn/hcnerrors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hcn/hcnerrors.go b/hcn/hcnerrors.go index e7a850b29c..6b12bfdae0 100644 --- a/hcn/hcnerrors.go +++ b/hcn/hcnerrors.go @@ -62,7 +62,7 @@ type EndpointNotFoundError struct { } func (e EndpointNotFoundError) Error() string { - if e.EndpointName == "" { + if e.EndpointName != "" { return fmt.Sprintf("Endpoint Name %s not found", e.EndpointName) } return fmt.Sprintf("Endpoint Id %s not found", e.EndpointID) From c1966bdeb09f1888810ea08fee32cd8392b77d54 Mon Sep 17 00:00:00 2001 From: Johnson Shih Date: Mon, 26 Jul 2021 18:05:41 -0700 Subject: [PATCH 22/33] Add shutdown option parameter (#10) --- cmd/containerd-shim-runhcs-v1/task_hcs.go | 2 +- cmd/runhcs/shim.go | 2 +- internal/cow/cow.go | 2 +- internal/gcs/container.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs.go b/cmd/containerd-shim-runhcs-v1/task_hcs.go index 6815632c9f..3216dc9689 100644 --- a/cmd/containerd-shim-runhcs-v1/task_hcs.go +++ b/cmd/containerd-shim-runhcs-v1/task_hcs.go @@ -520,7 +520,7 @@ func (ht *hcsTask) close(ctx context.Context) { werr = ht.c.Wait() close(ch) }() - err := ht.c.Shutdown(ctx) + err := ht.c.Shutdown(ctx, "") if err != nil { log.G(ctx).WithError(err).Error("failed to shutdown container") } else { diff --git a/cmd/runhcs/shim.go b/cmd/runhcs/shim.go index 0d052611f5..9bb5561e99 100644 --- a/cmd/runhcs/shim.go +++ b/cmd/runhcs/shim.go @@ -214,7 +214,7 @@ var shimCommand = cli.Command{ // Shutdown the container, waiting 5 minutes before terminating is // forcefully. const shutdownTimeout = time.Minute * 5 - err := c.hc.Shutdown(gcontext.Background()) + err := c.hc.Shutdown(gcontext.Background(), "") if err != nil { select { case <-containerExitCh: diff --git a/internal/cow/cow.go b/internal/cow/cow.go index c0158269f0..67d4f9bee3 100644 --- a/internal/cow/cow.go +++ b/internal/cow/cow.go @@ -69,7 +69,7 @@ type Container interface { Start(ctx context.Context) error // Shutdown sends a shutdown request to the container (but does not wait for // the shutdown to complete). - Shutdown(ctx context.Context) error + Shutdown(ctx context.Context, options string) error // Terminate sends a terminate request to the container (but does not wait // for the terminate to complete). Terminate(ctx context.Context) error diff --git a/internal/gcs/container.go b/internal/gcs/container.go index 58a2b65e74..624179cf3a 100644 --- a/internal/gcs/container.go +++ b/internal/gcs/container.go @@ -136,7 +136,7 @@ func (c *Container) shutdown(ctx context.Context, proc rpcProc) error { // Shutdown sends a graceful shutdown request to the container. The container // might not be terminated by the time the request completes (and might never // terminate). -func (c *Container) Shutdown(ctx context.Context) error { +func (c *Container) Shutdown(ctx context.Context, options string) error { return c.shutdown(ctx, rpcShutdownGraceful) } From 648cac11449f1306da2cb60b628567b24f57a3c0 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Wed, 21 Apr 2021 16:08:37 -0700 Subject: [PATCH 23/33] Add UpdateGpuConfiguration for GPU-PV --- internal/requesttype/types.go | 1 + internal/schema2/gpu_configuration.go | 24 ++++++++++++++++ vm.go | 41 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 internal/schema2/gpu_configuration.go diff --git a/internal/requesttype/types.go b/internal/requesttype/types.go index 8c0e1b4555..df2db709b0 100644 --- a/internal/requesttype/types.go +++ b/internal/requesttype/types.go @@ -7,4 +7,5 @@ const ( Add = "Add" Remove = "Remove" PreAdd = "PreAdd" // For networking + Update = "Update" ) diff --git a/internal/schema2/gpu_configuration.go b/internal/schema2/gpu_configuration.go new file mode 100644 index 0000000000..f71792a498 --- /dev/null +++ b/internal/schema2/gpu_configuration.go @@ -0,0 +1,24 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type GpuConfiguration struct { + + // The mode used to assign GPUs to the guest. + AssignmentMode string `json:"AssignmentMode,omitempty"` + + // This only applies to List mode, and is ignored in other modes. + // In GPU-P, string is GPU device interface, and unit16 is partition id. HCS simply assigns the partition with the input id. + // In GPU-PV, string is GPU device interface, and unit16 is 0xffff. HCS needs to find an available partition to assign. + AssignmentRequest map[string]uint16 `json:"AssignmentRequest,omitempty"` + + // Whether we allow vendor extension. + AllowVendorExtension bool `json:"AllowVendorExtension,omitempty"` +} diff --git a/vm.go b/vm.go index ab96231cc7..82778a8306 100644 --- a/vm.go +++ b/vm.go @@ -17,6 +17,15 @@ import ( "github.com/Microsoft/hcsshim/osversion" ) +type GpuAssignmentMode string + +const ( + GpuAssignmentModeDisabled = GpuAssignmentMode("Disabled") + GpuAssignmentModeDefault = GpuAssignmentMode("Default") + GpuAssignmentModeList = GpuAssignmentMode("List") + GpuAssignmentModeMirror = GpuAssignmentMode("Mirror") +) + type VirtualMachineOptions struct { Name string Id string @@ -455,6 +464,38 @@ func (vm *VirtualMachineSpec) RemovePlan9(shareName string, uvmPath string) (err return nil } +func (vm *VirtualMachineSpec) UpdateGpuConfiguration(mode GpuAssignmentMode, allowVendorExtension bool, assignments map[string]uint16) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + defer system.Close() + + settings := hcsschema.GpuConfiguration{ + AssignmentMode: string(mode), + AllowVendorExtension: allowVendorExtension, + } + + if len(assignments) != 0 { + settings.AssignmentRequest = assignments + } + + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Update, + ResourcePath: "VirtualMachine/ComputeTopology/Gpu", + Settings: settings, + } + + if err := system.Modify(ctx, request); err != nil { + return err + } + + return nil +} + func (vm *VirtualMachineSpec) String() string { jsonString, err := json.Marshal(vm.spec) if err != nil { From bb5d7249e4c0772b019e57541d8c07abd7350ae4 Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 29 Apr 2021 09:41:07 -0700 Subject: [PATCH 24/33] Grant Vm access to vhd and iso files --- vm.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vm.go b/vm.go index 82778a8306..70b213defa 100644 --- a/vm.go +++ b/vm.go @@ -14,6 +14,7 @@ import ( "github.com/Microsoft/hcsshim/internal/requesttype" "github.com/Microsoft/hcsshim/internal/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/schema2" + "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" ) @@ -50,6 +51,14 @@ type VirtualMachineSpec struct { } func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, error) { + // Ensure the VM has access, we use opts.Id to create VM + if err := wclayer.GrantVmAccess(opts.Id, opts.VhdPath); err != nil { + return nil, err + } + if err := wclayer.GrantVmAccess(opts.Id, opts.IsoPath); err != nil { + return nil, err + } + spec := &hcsschema.ComputeSystem{ Owner: opts.Owner, SchemaVersion: &hcsschema.Version{ From bb6bcc33096b038b3bb9994fecde1669b7cab84d Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 29 Apr 2021 14:40:37 -0700 Subject: [PATCH 25/33] Remove redundant type --- vm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm.go b/vm.go index 70b213defa..a1cd0f96d5 100644 --- a/vm.go +++ b/vm.go @@ -86,13 +86,13 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, }, Devices: &hcsschema.Devices{ Scsi: map[string]hcsschema.Scsi{ - "primary": hcsschema.Scsi{ + "primary": { Attachments: map[string]hcsschema.Attachment{ - "0": hcsschema.Attachment{ + "0": { Path: opts.VhdPath, Type_: "VirtualDisk", }, - "1": hcsschema.Attachment{ + "1": { Path: opts.IsoPath, Type_: "Iso", }, From 573ae700d97f6bee38c057e4cdafc73ce96b593f Mon Sep 17 00:00:00 2001 From: Johnson Shih Date: Fri, 11 Jun 2021 14:11:14 -0700 Subject: [PATCH 26/33] skip if notificationData is null --- hcn/hcncallback.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hcn/hcncallback.go b/hcn/hcncallback.go index 87cc169d3d..d7f1fd193e 100644 --- a/hcn/hcncallback.go +++ b/hcn/hcncallback.go @@ -105,6 +105,10 @@ func notificationWatcher(notificationType HcnNotification, callbackNumber uintpt return 0 } + if notificationData == nil { + return 0 + } + tmpString := syscall.UTF16ToString((*[1 << 29]uint16)(unsafe.Pointer(notificationData))[:]) var notifData NotificationBase From 11c66164b9ddfbefcb35840ece66f7eee5c62ecd Mon Sep 17 00:00:00 2001 From: Johnson Shih Date: Thu, 22 Jul 2021 17:47:31 -0700 Subject: [PATCH 27/33] correct logic for EndpointNotFoundError --- hcn/hcnerrors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hcn/hcnerrors.go b/hcn/hcnerrors.go index e7a850b29c..6b12bfdae0 100644 --- a/hcn/hcnerrors.go +++ b/hcn/hcnerrors.go @@ -62,7 +62,7 @@ type EndpointNotFoundError struct { } func (e EndpointNotFoundError) Error() string { - if e.EndpointName == "" { + if e.EndpointName != "" { return fmt.Sprintf("Endpoint Name %s not found", e.EndpointName) } return fmt.Sprintf("Endpoint Id %s not found", e.EndpointID) From 5a94048a66f2e9e35998cc91f2b9fa3c0f133d4f Mon Sep 17 00:00:00 2001 From: Cheng-Hsun Shih Date: Thu, 29 Apr 2021 16:02:34 -0700 Subject: [PATCH 28/33] Add ShutdownOptions support --- cmd/containerd-shim-runhcs-v1/task_hcs.go | 2 +- cmd/runhcs/shim.go | 2 +- container.go | 2 +- internal/cow/cow.go | 2 +- internal/gcs/container.go | 2 +- internal/hcs/system.go | 4 +-- internal/schema2/shutdown_options.go | 25 ++++++++++++++ vm.go | 41 +++++++++++++++++++++-- 8 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 internal/schema2/shutdown_options.go diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs.go b/cmd/containerd-shim-runhcs-v1/task_hcs.go index 6815632c9f..3216dc9689 100644 --- a/cmd/containerd-shim-runhcs-v1/task_hcs.go +++ b/cmd/containerd-shim-runhcs-v1/task_hcs.go @@ -520,7 +520,7 @@ func (ht *hcsTask) close(ctx context.Context) { werr = ht.c.Wait() close(ch) }() - err := ht.c.Shutdown(ctx) + err := ht.c.Shutdown(ctx, "") if err != nil { log.G(ctx).WithError(err).Error("failed to shutdown container") } else { diff --git a/cmd/runhcs/shim.go b/cmd/runhcs/shim.go index 0d052611f5..9bb5561e99 100644 --- a/cmd/runhcs/shim.go +++ b/cmd/runhcs/shim.go @@ -214,7 +214,7 @@ var shimCommand = cli.Command{ // Shutdown the container, waiting 5 minutes before terminating is // forcefully. const shutdownTimeout = time.Minute * 5 - err := c.hc.Shutdown(gcontext.Background()) + err := c.hc.Shutdown(gcontext.Background(), "") if err != nil { select { case <-containerExitCh: diff --git a/container.go b/container.go index 53c0a3854a..bc60e0a857 100644 --- a/container.go +++ b/container.go @@ -104,7 +104,7 @@ func (container *container) Start() error { // Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds. func (container *container) Shutdown() error { - err := container.system.Shutdown(context.Background()) + err := container.system.Shutdown(context.Background(), "") if err != nil { return convertSystemError(err, container) } diff --git a/internal/cow/cow.go b/internal/cow/cow.go index c0158269f0..67d4f9bee3 100644 --- a/internal/cow/cow.go +++ b/internal/cow/cow.go @@ -69,7 +69,7 @@ type Container interface { Start(ctx context.Context) error // Shutdown sends a shutdown request to the container (but does not wait for // the shutdown to complete). - Shutdown(ctx context.Context) error + Shutdown(ctx context.Context, options string) error // Terminate sends a terminate request to the container (but does not wait // for the terminate to complete). Terminate(ctx context.Context) error diff --git a/internal/gcs/container.go b/internal/gcs/container.go index 58a2b65e74..624179cf3a 100644 --- a/internal/gcs/container.go +++ b/internal/gcs/container.go @@ -136,7 +136,7 @@ func (c *Container) shutdown(ctx context.Context, proc rpcProc) error { // Shutdown sends a graceful shutdown request to the container. The container // might not be terminated by the time the request completes (and might never // terminate). -func (c *Container) Shutdown(ctx context.Context) error { +func (c *Container) Shutdown(ctx context.Context, options string) error { return c.shutdown(ctx, rpcShutdownGraceful) } diff --git a/internal/hcs/system.go b/internal/hcs/system.go index e1cb78a6fb..274b43f5a3 100644 --- a/internal/hcs/system.go +++ b/internal/hcs/system.go @@ -316,7 +316,7 @@ func (computeSystem *System) ID() string { } // Shutdown requests a compute system shutdown. -func (computeSystem *System) Shutdown(ctx gcontext.Context) (err error) { +func (computeSystem *System) Shutdown(ctx gcontext.Context, options string) (err error) { computeSystem.handleLock.RLock() defer computeSystem.handleLock.RUnlock() @@ -331,7 +331,7 @@ func (computeSystem *System) Shutdown(ctx gcontext.Context) (err error) { } var resultp *uint16 - err = hcsShutdownComputeSystemContext(ctx, computeSystem.handle, "", &resultp) + err = hcsShutdownComputeSystemContext(ctx, computeSystem.handle, options, &resultp) events := processHcsResult(ctx, resultp) switch err { case nil, ErrVmcomputeAlreadyStopped, ErrComputeSystemDoesNotExist, ErrVmcomputeOperationPending: diff --git a/internal/schema2/shutdown_options.go b/internal/schema2/shutdown_options.go new file mode 100644 index 0000000000..7ab7c7bea8 --- /dev/null +++ b/internal/schema2/shutdown_options.go @@ -0,0 +1,25 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ShutdownOptions struct { + + // What kind of mechanism used to perform the shutdown operation + Mechanism string `json:"Mechanism,omitempty"` + + // What is the type of the shutdown operation + Type string `json:"Type,omitempty"` + + // If this shutdown is forceful or not + Force bool `json:"Force,omitempty"` + + // Reason for the shutdown + Reason string `json:"Reason,omitempty"` +} diff --git a/vm.go b/vm.go index a1cd0f96d5..186a6939d8 100644 --- a/vm.go +++ b/vm.go @@ -27,6 +27,21 @@ const ( GpuAssignmentModeMirror = GpuAssignmentMode("Mirror") ) +type ShutdownMechanism string + +const ( + ShutdownMechanismGuestConnection = ShutdownMechanism("GuestConnection") + ShutdownMechanismIntegrationService = ShutdownMechanism("IntegrationService") +) + +type ShutdownType string + +const ( + ShutdownTypeShutdown = ShutdownType("Shutdown") + ShutdownTypeHibernate = ShutdownType("Hibernate") + ShutdownTypeReboot = ShutdownType("Reboot") +) + type VirtualMachineOptions struct { Name string Id string @@ -255,7 +270,7 @@ func (vm *VirtualMachineSpec) Start() error { } // Stop a Virtual Machine -func (vm *VirtualMachineSpec) Stop() error { +func (vm *VirtualMachineSpec) Stop(force bool) error { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() system, err := hcs.OpenComputeSystem(ctx, vm.ID) @@ -264,7 +279,12 @@ func (vm *VirtualMachineSpec) Stop() error { } defer system.Close() - return system.Shutdown(ctx) + shutdownOptions, err := generateShutdownOptions(force) + if err != nil { + return err + } + + return system.Shutdown(ctx, shutdownOptions) } // Delete a Virtual Machine @@ -513,3 +533,20 @@ func (vm *VirtualMachineSpec) String() string { return string(jsonString) } + +func generateShutdownOptions(force bool) (string, error) { + // TODO: shutdown options only supported at schema version 2.5 and above + // check current schema version on the running system and return empty string if + // running on schema version 2.4 and below + options := hcsschema.ShutdownOptions{ + Mechanism: string(ShutdownMechanismGuestConnection), + Type: string(ShutdownTypeShutdown), + Force: force, + Reason: "Requested shutdown", + } + optionsB, err := json.Marshal(options) + if err != nil { + return "", err + } + return string(optionsB), nil +} From c18e3d4f6172f5afdd5362e1f39b6dd185b1382c Mon Sep 17 00:00:00 2001 From: Jagadish Murugan Date: Fri, 28 May 2021 12:28:10 -0700 Subject: [PATCH 29/33] Add API to Add/Remove virtual device * Add API to Add/Remove virtual device * fix review comment --- internal/guestrequest/types.go | 5 ++ internal/schema2/devices.go | 4 ++ internal/schema2/virtual_pci_device.go | 16 ++++++ internal/schema2/virtual_pci_function.go | 18 +++++++ vm.go | 69 +++++++++++++++++++++++- 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 internal/schema2/virtual_pci_device.go create mode 100644 internal/schema2/virtual_pci_function.go diff --git a/internal/guestrequest/types.go b/internal/guestrequest/types.go index 3dafbdc00a..8ce139cc18 100644 --- a/internal/guestrequest/types.go +++ b/internal/guestrequest/types.go @@ -49,6 +49,10 @@ type LCOWMappedVPMemDevice struct { MountPath string `json:"MountPath,omitempty"` // /tmp/pN } +type LCOWMappedVPCIDevice struct { + VMBusGUID string `json:"VMBusGUID,omitempty"` +} + type LCOWNetworkAdapter struct { NamespaceID string `json:",omitempty"` ID string `json:",omitempty"` @@ -72,6 +76,7 @@ const ( ResourceTypeNetworkNamespace ResourceType = "NetworkNamespace" ResourceTypeCombinedLayers ResourceType = "CombinedLayers" ResourceTypeVPMemDevice ResourceType = "VPMemDevice" + ResourceTypeVPCIDevice ResourceType = "VPCIDevice" ) // GuestRequest is for modify commands passed to the guest. diff --git a/internal/schema2/devices.go b/internal/schema2/devices.go index 781a884015..e985d96d22 100644 --- a/internal/schema2/devices.go +++ b/internal/schema2/devices.go @@ -39,4 +39,8 @@ type Devices struct { FlexibleIov map[string]FlexibleIoDevice `json:"FlexibleIov,omitempty"` SharedMemory *SharedMemoryConfiguration `json:"SharedMemory,omitempty"` + + // TODO: This is pre-release support in schema 2.3. Need to add build number + // docs when a public build with this is out. + VirtualPci map[string]VirtualPciDevice `json:",omitempty"` } diff --git a/internal/schema2/virtual_pci_device.go b/internal/schema2/virtual_pci_device.go new file mode 100644 index 0000000000..f5e05903c5 --- /dev/null +++ b/internal/schema2/virtual_pci_device.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.3 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// TODO: This is pre-release support in schema 2.3. Need to add build number +// docs when a public build with this is out. +type VirtualPciDevice struct { + Functions []VirtualPciFunction `json:",omitempty"` +} diff --git a/internal/schema2/virtual_pci_function.go b/internal/schema2/virtual_pci_function.go new file mode 100644 index 0000000000..cedb7d18bc --- /dev/null +++ b/internal/schema2/virtual_pci_function.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.3 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// TODO: This is pre-release support in schema 2.3. Need to add build number +// docs when a public build with this is out. +type VirtualPciFunction struct { + DeviceInstancePath string `json:",omitempty"` + + VirtualFunction uint16 `json:",omitempty"` +} diff --git a/vm.go b/vm.go index 186a6939d8..367334bc5e 100644 --- a/vm.go +++ b/vm.go @@ -8,6 +8,7 @@ import ( "path" "time" + "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/internal/guestrequest" "github.com/Microsoft/hcsshim/internal/hcs" @@ -53,6 +54,7 @@ type VirtualMachineOptions struct { VnicId string MacAddress string UseGuestConnection bool + AllowOvercommit bool } const plan9Port = 564 @@ -93,7 +95,8 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, }, ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.Memory2{ - SizeInMB: int32(opts.MemoryInMB), + SizeInMB: int32(opts.MemoryInMB), + AllowOvercommit: opts.AllowOvercommit, }, Processor: &hcsschema.Processor2{ Count: int32(opts.ProcessorCount), @@ -525,6 +528,70 @@ func (vm *VirtualMachineSpec) UpdateGpuConfiguration(mode GpuAssignmentMode, all return nil } +// Add vPCI device +func (vm *VirtualMachineSpec) AssignDevice(ctx context.Context, deviceID string) (string, error) { + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return "", err + } + defer system.Close() + + guid, err := guid.NewV4() + if err != nil { + return "", err + } + + vmBusGUID := guid.String() + targetDevice := hcsschema.VirtualPciDevice{ + Functions: []hcsschema.VirtualPciFunction{ + { + DeviceInstancePath: deviceID, + }, + }, + } + request := &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPci/%s", vmBusGUID), + RequestType: requesttype.Add, + Settings: targetDevice, + } + + // for LCOW, we need to make sure that specific paths relating to the + // device exist so they are ready to be used by later + // work in openGCS + request.GuestRequest = guestrequest.GuestRequest{ + ResourceType: guestrequest.ResourceTypeVPCIDevice, + RequestType: requesttype.Add, + Settings: guestrequest.LCOWMappedVPCIDevice{ + VMBusGUID: vmBusGUID, + }, + } + + if err := system.Modify(ctx, request); err != nil { + return "", err + } + + return vmBusGUID, nil +} + +// Removes a vpci device from VirtualMachineSpec +func (vm *VirtualMachineSpec) RemoveDevice(ctx context.Context, vmBusGUID string) error { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return err + } + + defer system.Close() + + return system.Modify(ctx, &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf("VirtualMachine/Devices/VirtualPci/%s", vmBusGUID), + RequestType: requesttype.Remove, + }) + return nil +} + func (vm *VirtualMachineSpec) String() string { jsonString, err := json.Marshal(vm.spec) if err != nil { From 9ee216c9edf3aeef101a405aafff5d5bb4de0ed9 Mon Sep 17 00:00:00 2001 From: Jagadish Murugan Date: Thu, 27 Jan 2022 11:19:23 -0800 Subject: [PATCH 30/33] =?UTF-8?q?Query=20for=20specific=20network=20instea?= =?UTF-8?q?d=20of=20enumerating=20all=20networks=20and=20fi=E2=80=A6=20(#1?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Query for specific network instead of enumerating all networks and filtering. DefaultSwitch is created when hcnOpenNetwork is called for the first time * remove redundant check --- hcn/hcnerrors.go | 2 +- hcn/hcnnetwork.go | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hcn/hcnerrors.go b/hcn/hcnerrors.go index 6b12bfdae0..eeeb5608d6 100644 --- a/hcn/hcnerrors.go +++ b/hcn/hcnerrors.go @@ -49,7 +49,7 @@ type NetworkNotFoundError struct { } func (e NetworkNotFoundError) Error() string { - if e.NetworkName == "" { + if e.NetworkName != "" { return fmt.Sprintf("Network Name %s not found", e.NetworkName) } return fmt.Sprintf("Network Id %s not found", e.NetworkID) diff --git a/hcn/hcnnetwork.go b/hcn/hcnnetwork.go index b85cbafc1c..7d42d46bbc 100644 --- a/hcn/hcnnetwork.go +++ b/hcn/hcnnetwork.go @@ -415,15 +415,19 @@ func GetNetworkByID(networkID string) (*HostComputeNetwork, error) { return nil, err } hcnQuery.Filter = string(filter) + queryJson, _ := json.Marshal(hcnQuery) - networks, err := ListNetworksQuery(hcnQuery) + networkGuid, err := guid.FromString(networkID) if err != nil { return nil, err } - if len(networks) == 0 { - return nil, NetworkNotFoundError{NetworkID: networkID} + + network, err := getNetwork(networkGuid, string(queryJson)) + if err != nil { + return nil, err } - return &networks[0], err + + return network, err } // GetNetworkByName returns the network specified by Name. From 6cd66d98a186c9181865d432a402c03887f9fb3f Mon Sep 17 00:00:00 2001 From: Jagadish Murugan Date: Sat, 12 Mar 2022 23:01:24 -0800 Subject: [PATCH 31/33] Implement HCNNetworkWatcher (#11) * Implement HCNNetworkWatcher * fix review comments --- hcn/hcnnetwork.go | 109 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/hcn/hcnnetwork.go b/hcn/hcnnetwork.go index 7d42d46bbc..91bfa905f3 100644 --- a/hcn/hcnnetwork.go +++ b/hcn/hcnnetwork.go @@ -123,6 +123,13 @@ type HcnServiceWatcher struct { started bool } +type HcnNetworkWatcher struct { + handleLock sync.RWMutex + callbackNumber uintptr + watcherContext *notifcationWatcherContext + started bool +} + func NewHcnServiceWatcher() *HcnServiceWatcher { watcherContext := ¬ifcationWatcherContext{ channel: make(notificationChannel, 1), @@ -213,6 +220,108 @@ func (w *HcnServiceWatcher) Close() error { return nil } +func NewHcnNetworkWatcher() *HcnNetworkWatcher { + watcherContext := ¬ifcationWatcherContext{ + channel: make(notificationChannel, 1), + } + + callbackMapLock.Lock() + callbackNumber := nextCallback + nextCallback++ + callbackMap[callbackNumber] = watcherContext + callbackMapLock.Unlock() + + return &HcnNetworkWatcher{ + callbackNumber: callbackNumber, + watcherContext: watcherContext, + } +} + +func (w *HcnNetworkWatcher) Start(networkId string) error { + w.handleLock.RLock() + defer w.handleLock.RUnlock() + + if w.started { + return nil + } + + var networkHandle hcnNetwork + var resultBuffer *uint16 + networkGuid, err := guid.FromString(networkId) + if err != nil { + return errInvalidNetworkID + } + + hr := hcnOpenNetwork(&networkGuid, &networkHandle, &resultBuffer) + if err = checkForErrors("hcnOpenNetwork", hr, resultBuffer); err != nil { + return err + } + + var callbackHandle hcnCallbackHandle + err = hcnRegisterNetworkCallback(networkHandle, notificationWatcherCallback, w.callbackNumber, &callbackHandle) + if err != nil { + return err + } + + w.watcherContext.handle = callbackHandle + w.started = true + return nil +} + +func (w *HcnNetworkWatcher) Stop() error { + var handle hcnCallbackHandle + + { + w.handleLock.RLock() + defer w.handleLock.RUnlock() + + if !w.started { + return nil + } + w.started = false + handle = w.watcherContext.handle + w.watcherContext.handle = 0 + } + + if handle == 0 { + return nil + } + + // hcnUnregisterNetworkCallback has its own syncronization + // to wait for all callbacks to complete. We must NOT hold the callbackMapLock. + err := hcnUnregisterNetworkCallback(handle) + if err != nil { + return err + } + + return nil +} + +func (w *HcnNetworkWatcher) Notification() <-chan HcnNotificationData { + w.handleLock.RLock() + defer w.handleLock.RUnlock() + + if !w.started { + return nil + } + return w.watcherContext.channel +} + +func (w *HcnNetworkWatcher) Close() error { + w.Stop() + + w.handleLock.RLock() + close(w.watcherContext.channel) + callbackNumber := w.callbackNumber + w.handleLock.RUnlock() + + callbackMapLock.Lock() + delete(callbackMap, callbackNumber) + callbackMapLock.Unlock() + + return nil +} + func getNetwork(networkGuid guid.GUID, query string) (*HostComputeNetwork, error) { // Open network. var ( From 742da338315b1a7e6b79deffd3653d4a8457d72e Mon Sep 17 00:00:00 2001 From: Johnson Shih Date: Fri, 18 Mar 2022 15:42:34 -0700 Subject: [PATCH 32/33] Add options to support secure boot and guest connection useVsock (#14) * Add secure boot support * Add GuestConnectionUseVsock option --- internal/schema2/uefi.go | 3 +++ vm.go | 56 ++++++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/internal/schema2/uefi.go b/internal/schema2/uefi.go index 0e48ece500..fd175ec1a0 100644 --- a/internal/schema2/uefi.go +++ b/internal/schema2/uefi.go @@ -14,6 +14,9 @@ type Uefi struct { SecureBootTemplateId string `json:"SecureBootTemplateId,omitempty"` + // schema version 2.3 + ApplySecureBootTemplate string `json:"ApplySecureBootTemplate,omitempty"` + BootThis *UefiBootEntry `json:"BootThis,omitempty"` Console string `json:"Console,omitempty"` diff --git a/vm.go b/vm.go index 367334bc5e..8b628f586e 100644 --- a/vm.go +++ b/vm.go @@ -44,17 +44,20 @@ const ( ) type VirtualMachineOptions struct { - Name string - Id string - VhdPath string - IsoPath string - Owner string - MemoryInMB int32 - ProcessorCount int32 - VnicId string - MacAddress string - UseGuestConnection bool - AllowOvercommit bool + Name string + Id string + VhdPath string + IsoPath string + Owner string + MemoryInMB int32 + ProcessorCount int32 + VnicId string + MacAddress string + UseGuestConnection bool + GuestConnectionUseVsock bool + AllowOvercommit bool + SecureBootEnabled bool + SecureBootTemplateId string } const plan9Port = 564 @@ -76,12 +79,12 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, return nil, err } + // determine which schema version to use + schemaVersion := getSchemaVersion(opts.SecureBootEnabled) + spec := &hcsschema.ComputeSystem{ - Owner: opts.Owner, - SchemaVersion: &hcsschema.Version{ - Major: 2, - Minor: 1, - }, + Owner: opts.Owner, + SchemaVersion: &schemaVersion, ShouldTerminateOnLastHandleClosed: true, VirtualMachine: &hcsschema.VirtualMachine{ Chipset: &hcsschema.Chipset{ @@ -132,11 +135,16 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, if opts.UseGuestConnection { spec.VirtualMachine.GuestConnection = &hcsschema.GuestConnection{ - UseVsock: true, + UseVsock: opts.GuestConnectionUseVsock, UseConnectedSuspend: true, } } + if opts.SecureBootEnabled { + spec.VirtualMachine.Chipset.Uefi.SecureBootTemplateId = opts.SecureBootTemplateId + spec.VirtualMachine.Chipset.Uefi.ApplySecureBootTemplate = "Apply" + } + return &VirtualMachineSpec{ spec: spec, ID: opts.Id, @@ -617,3 +625,17 @@ func generateShutdownOptions(force bool) (string, error) { } return string(optionsB), nil } + +func getSchemaVersion(secureBootEnabled bool) hcsschema.Version { + if secureBootEnabled { + return hcsschema.Version{ + Major: 2, + Minor: 3, + } + } + + return hcsschema.Version{ + Major: 2, + Minor: 1, + } +} From ffcf64a9a6bc20ee6a50190b33ec63c8f65a853b Mon Sep 17 00:00:00 2001 From: Johnson Shih Date: Wed, 21 Sep 2022 22:28:12 -0700 Subject: [PATCH 33/33] add mmio space configuration, RunCommand and query supported schema version api (#10) * Add high mmio space configuration (#15) * HCS API to run command on Guest * fix review comments * fix review comments * fix review comment * return exitcode for guest commands (#17) * api to query supported schema version (#18) * api to query supported schema version * fix review comments * handle multiple supported schema versions * fix review comments * check minor version as well * compatibility fix for go 1.16.15 (#19) Co-authored-by: Jagadish Murugan Co-authored-by: Sean Yen --- internal/hcs/service.go | 39 ++++++++++ internal/schema2/basicinformation.go | 14 ++++ internal/schema2/memory_2.go | 4 ++ internal/schema2/service_properties.go | 18 +++++ service.go | 40 +++++++++++ vm.go | 98 +++++++++++++++++++++++++- 6 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 internal/hcs/service.go create mode 100644 internal/schema2/basicinformation.go create mode 100644 internal/schema2/service_properties.go create mode 100644 service.go diff --git a/internal/hcs/service.go b/internal/hcs/service.go new file mode 100644 index 0000000000..a306fb0c8e --- /dev/null +++ b/internal/hcs/service.go @@ -0,0 +1,39 @@ +// +build windows + +package hcs + +import ( + "context" + "encoding/json" + + "github.com/Microsoft/hcsshim/internal/interop" + hcsschema "github.com/Microsoft/hcsshim/internal/schema2" +) + +// GetServiceProperties returns properties of the host compute service. +func GetServiceProperties(ctx context.Context, q hcsschema.PropertyQuery) (properties *hcsschema.ServiceProperties, err error) { + operation := "hcs::GetServiceProperties" + + queryb, err := json.Marshal(q) + if err != nil { + return nil, err + } + + var resultp, propertiesp *uint16 + err = hcsGetServiceProperties(string(queryb), &propertiesp, &resultp) + events := processHcsResult(ctx, resultp) + if err != nil { + return nil, &HcsError{Op: operation, Err: err, Events: events} + } + + if propertiesp == nil { + return nil, ErrUnexpectedValue + } + propertiesRaw := interop.ConvertAndFreeCoTaskMemString(propertiesp) + properties = &hcsschema.ServiceProperties{} + if err := json.Unmarshal([]byte(propertiesRaw), properties); err != nil { + return nil, err + } + + return properties, nil +} diff --git a/internal/schema2/basicinformation.go b/internal/schema2/basicinformation.go new file mode 100644 index 0000000000..2b22ceed58 --- /dev/null +++ b/internal/schema2/basicinformation.go @@ -0,0 +1,14 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type BasicInformation struct { + SupportedSchemaVersions []Version `json:"SupportedSchemaVersions,omitempty"` +} diff --git a/internal/schema2/memory_2.go b/internal/schema2/memory_2.go index 27d0b8c483..54b8fefd4d 100644 --- a/internal/schema2/memory_2.go +++ b/internal/schema2/memory_2.go @@ -20,6 +20,10 @@ type Memory2 struct { EnableEpf bool `json:"EnableEpf,omitempty"` + HighMmioBaseInMB int32 `json:"HighMmioBaseInMB,omitempty"` + + HighMmioGapInMB int32 `json:"HighMmioGapInMB,omitempty"` + // EnableDeferredCommit is private in the schema. If regenerated need to add back. EnableDeferredCommit bool `json:"EnableDeferredCommit,omitempty"` } diff --git a/internal/schema2/service_properties.go b/internal/schema2/service_properties.go new file mode 100644 index 0000000000..79ed0d32f0 --- /dev/null +++ b/internal/schema2/service_properties.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +import "encoding/json" + +type ServiceProperties struct { + // Changed Properties field to []json.RawMessage from []interface{} to avoid having to + // remarshal sp.Properties[n] and unmarshal into the type(s) we want. + Properties []json.RawMessage `json:"Properties,omitempty"` +} diff --git a/service.go b/service.go new file mode 100644 index 0000000000..a0bcda72b5 --- /dev/null +++ b/service.go @@ -0,0 +1,40 @@ +package hcsshim + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/Microsoft/hcsshim/internal/hcs" + hcsschema "github.com/Microsoft/hcsshim/internal/schema2" +) + +func GetHighestSupportedHcsSchemaVersion() (schemaVersion *hcsschema.Version, err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + serviceProperties, err := hcs.GetServiceProperties(ctx, hcsschema.PropertyQuery{PropertyTypes: []string{"Basic"}}) + if err != nil { + return nil, fmt.Errorf("failed to retrieve HCS schema version: %s", err) + } + + basicInfo := &hcsschema.BasicInformation{} + if err := json.Unmarshal(serviceProperties.Properties[0], &basicInfo); err != nil { + return nil, fmt.Errorf("failed to unmarshal HCS Schema Version: %s", err) + } + + var highestVersion hcsschema.Version + for _, version := range basicInfo.SupportedSchemaVersions { + if version.Major >= highestVersion.Major { + if (version.Major > highestVersion.Major) || (version.Minor > highestVersion.Minor) { + highestVersion.Major = version.Major + highestVersion.Minor = version.Minor + } + } + } + + schemaVersion = &highestVersion + + return +} diff --git a/vm.go b/vm.go index 8b628f586e..ce7d73cc0a 100644 --- a/vm.go +++ b/vm.go @@ -1,11 +1,13 @@ package hcsshim import ( + "bytes" "context" "encoding/json" "errors" "fmt" "path" + "strings" "time" "github.com/Microsoft/go-winio/pkg/guid" @@ -17,6 +19,7 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/schema2" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + "golang.org/x/sys/windows" ) type GpuAssignmentMode string @@ -58,6 +61,8 @@ type VirtualMachineOptions struct { AllowOvercommit bool SecureBootEnabled bool SecureBootTemplateId string + HighMmioBaseInMB int32 + HighMmioGapInMB int32 } const plan9Port = 564 @@ -80,7 +85,7 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, } // determine which schema version to use - schemaVersion := getSchemaVersion(opts.SecureBootEnabled) + schemaVersion := getSchemaVersion(opts) spec := &hcsschema.ComputeSystem{ Owner: opts.Owner, @@ -145,6 +150,14 @@ func CreateVirtualMachineSpec(opts *VirtualMachineOptions) (*VirtualMachineSpec, spec.VirtualMachine.Chipset.Uefi.ApplySecureBootTemplate = "Apply" } + if opts.HighMmioBaseInMB != 0 { + spec.VirtualMachine.ComputeTopology.Memory.HighMmioBaseInMB = opts.HighMmioBaseInMB + } + + if opts.HighMmioGapInMB != 0 { + spec.VirtualMachine.ComputeTopology.Memory.HighMmioGapInMB = opts.HighMmioGapInMB + } + return &VirtualMachineSpec{ spec: spec, ID: opts.Id, @@ -341,6 +354,85 @@ func (vm *VirtualMachineSpec) ExecuteCommand(command string) error { return nil } +// escapeArgs makes a Windows-style escaped command line from a set of arguments +func escapeArgs(args []string) string { + escapedArgs := make([]string, len(args)) + for i, a := range args { + escapedArgs[i] = windows.EscapeArg(a) + } + return strings.Join(escapedArgs, " ") +} + +// RunCommand executes a command on the Virtual Machine +func (vm *VirtualMachineSpec) RunCommand(command []string, user string) (exitCode int, output string, errOut string, err error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + exitCode = 1 + + system, err := hcs.OpenComputeSystem(ctx, vm.ID) + if err != nil { + return exitCode, "", "", err + } + defer system.Close() + + var params *hcsschema.ProcessParameters + switch system.OS() { + case "linux": + params = &hcsschema.ProcessParameters{ + CommandArgs: command, + WorkingDirectory: "/", + User: user, + Environment: map[string]string{"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + CreateStdInPipe: false, + CreateStdOutPipe: true, + CreateStdErrPipe: true, + ConsoleSize: []int32{0, 0}, + } + case "windows": + params = &hcsschema.ProcessParameters{ + CommandLine: escapeArgs(command), + WorkingDirectory: `C:\`, + User: user, + CreateStdInPipe: false, + CreateStdOutPipe: true, + CreateStdErrPipe: true, + ConsoleSize: []int32{0, 0}, + } + default: + return exitCode, "", "", ErrNotSupported + } + + process, err := system.CreateProcess(ctx, params) + if err != nil { + return + } + + defer process.Close() + + err = process.Wait() + if err != nil { + return exitCode, "Wait returned error!", "", err + } + + exitCode, err = process.ExitCode() + + _, reader, errReader := process.Stdio() + if reader != nil { + outBuf := new(bytes.Buffer) + outBuf.ReadFrom(reader) + output = strings.TrimSpace(outBuf.String()) + } + + if errReader != nil { + errBuf := new(bytes.Buffer) + errBuf.ReadFrom(errReader) + errOut = strings.TrimSpace(errBuf.String()) + } + + return +} + func (vm *VirtualMachineSpec) HotAttachEndpoints(endpoints []*hcn.HostComputeEndpoint) (err error) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) defer cancel() @@ -626,8 +718,8 @@ func generateShutdownOptions(force bool) (string, error) { return string(optionsB), nil } -func getSchemaVersion(secureBootEnabled bool) hcsschema.Version { - if secureBootEnabled { +func getSchemaVersion(opts *VirtualMachineOptions) hcsschema.Version { + if opts.SecureBootEnabled || opts.HighMmioBaseInMB != 0 || opts.HighMmioGapInMB != 0 { return hcsschema.Version{ Major: 2, Minor: 3,