From fbd69710b682e67806f64e7e5e437c2b305b2aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bojanowski?= Date: Mon, 4 Apr 2022 14:34:30 +0200 Subject: [PATCH 01/14] update gcp regions --- internal/cloudinfo/providers/google/cloudinfo.go | 11 +++++++++++ 1 file changed, 11 insertions(+) mode change 100644 => 100755 internal/cloudinfo/providers/google/cloudinfo.go diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go old mode 100644 new mode 100755 index 34bd892b0..ce70a7385 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -41,21 +41,32 @@ var regionNames = map[string]string{ "asia-east1": "Asia Pacific (Taiwan)", "asia-east2": "Asia Pacific (Hong Kong)", "asia-northeast1": "Asia Pacific (Tokyo)", + "asia-northeast2": "Asia Pacific (Osaka)", + "asia-northeast3": "Asia Pacific (Seoul)", "asia-south1": "Asia Pacific (Mumbai)", + "asia-south2": "Asia Pacific (Delhi)", "asia-southeast1": "Asia Pacific (Singapore)", + "asia-southeast2": "Asia Pacific (Jakarta)", "australia-southeast1": "Asia Pacific (Sydney)", + "australia-southeast2": "Asia Pacific (Melbourne)", "europe-north1": "EU (Finland)", + "europe-central2": "EU (Warsaw)", "europe-west1": "EU (Belgium)", "europe-west2": "EU (London)", "europe-west3": "EU (Frankfurt)", "europe-west4": "EU (Netherlands)", + "europe-west6": "EU (Zurich)", "northamerica-northeast1": "Canada (Montréal)", + "northamerica-northeast2": "Canada (Toronto)", "southamerica-east1": "South America (São Paulo)", + "southamerica-west1": "South America (Santiago)", "us-central1": "US Central (Iowa)", "us-east1": "US East (South Carolina)", "us-east4": "US East (Northern Virginia)", "us-west1": "US West (Oregon)", "us-west2": "US West (Los Angeles)", + "us-west3": "US West (Salt Lake City)", + "us-west4": "US West (Las Vegas)", } // GceInfoer encapsulates the data and operations needed to access external resources From 42b7a0f48a8d2268e325a30ee3b866d29e6d292a Mon Sep 17 00:00:00 2001 From: Maciek Urbanski Date: Mon, 25 Apr 2022 10:13:52 +0200 Subject: [PATCH 02/14] Use map instead of iterating over and over again price list. --- .../cloudinfo/providers/google/cloudinfo.go | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index ce70a7385..916bf82ba 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -197,35 +197,36 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { return nil, err } zonesInRegions[r] = zones + err = g.computeSvc.MachineTypes.List(g.projectId, zones[0]).Pages(context.TODO(), func(allMts *compute.MachineTypeList) error { - for region, price := range pricePerRegion { - for _, mt := range allMts.Items { - if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) { - if allPrices[region] == nil { - allPrices[region] = make(map[string]types.Price) - } - prices := allPrices[region][mt.Name] + region := r + price := pricePerRegion[region] + for _, mt := range allMts.Items { + if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) { + if allPrices[region] == nil { + allPrices[region] = make(map[string]types.Price) + } + prices := allPrices[region][mt.Name] + if mt.Name == "f1-micro" || mt.Name == "g1-small" { + prices.OnDemandPrice = price[mt.Name]["OnDemand"] + } else { + prices.OnDemandPrice = price[types.CPU]["OnDemand"]*float64(mt.GuestCpus) + price[types.Memory]["OnDemand"]*float64(mt.MemoryMb)/1024 + } + spotPrice := make(types.SpotPriceInfo) + for _, z := range zonesInRegions[region] { if mt.Name == "f1-micro" || mt.Name == "g1-small" { - prices.OnDemandPrice = price[mt.Name]["OnDemand"] - } else { - prices.OnDemandPrice = price[types.CPU]["OnDemand"]*float64(mt.GuestCpus) + price[types.Memory]["OnDemand"]*float64(mt.MemoryMb)/1024 - } - spotPrice := make(types.SpotPriceInfo) - for _, z := range zonesInRegions[region] { - if mt.Name == "f1-micro" || mt.Name == "g1-small" { - spotPrice[z] = price[mt.Name]["Preemptible"] - metrics.ReportGoogleSpotPrice(region, z, mt.Name, spotPrice[z]) - } else { - spotPrice[z] = price[types.CPU]["Preemptible"]*float64(mt.GuestCpus) + price[types.Memory]["Preemptible"]*float64(mt.MemoryMb)/1024 - } - + spotPrice[z] = price[mt.Name]["Preemptible"] metrics.ReportGoogleSpotPrice(region, z, mt.Name, spotPrice[z]) + } else { + spotPrice[z] = price[types.CPU]["Preemptible"]*float64(mt.GuestCpus) + price[types.Memory]["Preemptible"]*float64(mt.MemoryMb)/1024 } - prices.SpotPrice = spotPrice - allPrices[region][mt.Name] = prices + metrics.ReportGoogleSpotPrice(region, z, mt.Name, spotPrice[z]) } + prices.SpotPrice = spotPrice + + allPrices[region][mt.Name] = prices } } return nil From 9b7cac3cb79a71a3c54073a60f2df7d6d2f72917 Mon Sep 17 00:00:00 2001 From: Maciek Urbanski Date: Mon, 25 Apr 2022 10:40:42 +0200 Subject: [PATCH 03/14] Switch google from regional to zonal data --- internal/app/cloudinfo/api/validate.go | 5 +- .../cloudinfo/providers/google/cloudinfo.go | 56 ++++++++++--------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/internal/app/cloudinfo/api/validate.go b/internal/app/cloudinfo/api/validate.go index a6295f9d1..d36008d43 100644 --- a/internal/app/cloudinfo/api/validate.go +++ b/internal/app/cloudinfo/api/validate.go @@ -15,6 +15,8 @@ package api import ( + "strings" + "emperror.dev/errors" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" @@ -67,7 +69,8 @@ func regionValidator(cpi types.CloudInfo, logger cloudinfo.Logger) validator.Fun } for reg := range regions { - if reg == regionPathParams.Region { + // contains covers validation of zones for google (zone == region + prefix, for ex. us-central1-b) + if reg == regionPathParams.Region || strings.Contains(regionPathParams.Region, reg) { return true } } diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index 916bf82ba..80b00d53b 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -198,41 +198,43 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { } zonesInRegions[r] = zones - err = g.computeSvc.MachineTypes.List(g.projectId, zones[0]).Pages(context.TODO(), func(allMts *compute.MachineTypeList) error { - region := r - price := pricePerRegion[region] - for _, mt := range allMts.Items { - if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) { - if allPrices[region] == nil { - allPrices[region] = make(map[string]types.Price) - } - prices := allPrices[region][mt.Name] + for _, zone := range zones { + err = g.computeSvc.MachineTypes.List(g.projectId, zone).Pages(context.TODO(), func(allMts *compute.MachineTypeList) error { + region := r + price := pricePerRegion[region] + for _, mt := range allMts.Items { + if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) { + if allPrices[zone] == nil { + allPrices[zone] = make(map[string]types.Price) + } + prices := allPrices[zone][mt.Name] - if mt.Name == "f1-micro" || mt.Name == "g1-small" { - prices.OnDemandPrice = price[mt.Name]["OnDemand"] - } else { - prices.OnDemandPrice = price[types.CPU]["OnDemand"]*float64(mt.GuestCpus) + price[types.Memory]["OnDemand"]*float64(mt.MemoryMb)/1024 - } - spotPrice := make(types.SpotPriceInfo) - for _, z := range zonesInRegions[region] { if mt.Name == "f1-micro" || mt.Name == "g1-small" { - spotPrice[z] = price[mt.Name]["Preemptible"] - metrics.ReportGoogleSpotPrice(region, z, mt.Name, spotPrice[z]) + prices.OnDemandPrice = price[mt.Name]["OnDemand"] } else { - spotPrice[z] = price[types.CPU]["Preemptible"]*float64(mt.GuestCpus) + price[types.Memory]["Preemptible"]*float64(mt.MemoryMb)/1024 + prices.OnDemandPrice = price[types.CPU]["OnDemand"]*float64(mt.GuestCpus) + price[types.Memory]["OnDemand"]*float64(mt.MemoryMb)/1024 } + spotPrice := make(types.SpotPriceInfo) + for _, z := range zonesInRegions[region] { + if mt.Name == "f1-micro" || mt.Name == "g1-small" { + spotPrice[z] = price[mt.Name]["Preemptible"] + metrics.ReportGoogleSpotPrice(region, z, mt.Name, spotPrice[z]) + } else { + spotPrice[z] = price[types.CPU]["Preemptible"]*float64(mt.GuestCpus) + price[types.Memory]["Preemptible"]*float64(mt.MemoryMb)/1024 + } - metrics.ReportGoogleSpotPrice(region, z, mt.Name, spotPrice[z]) - } - prices.SpotPrice = spotPrice + metrics.ReportGoogleSpotPrice(region, z, mt.Name, spotPrice[z]) + } + prices.SpotPrice = spotPrice - allPrices[region][mt.Name] = prices + allPrices[zone][mt.Name] = prices + } } + return nil + }) + if err != nil { + return nil, err } - return nil - }) - if err != nil { - return nil, err } } From 8a5d0df2fcf4fb5fd7781fcd6ec15ec91f0e6941 Mon Sep 17 00:00:00 2001 From: Maciek Urbanski Date: Mon, 25 Apr 2022 17:05:04 +0200 Subject: [PATCH 04/14] Save google vm types per zone in files --- internal/cloudinfo/scrape.go | 53 +++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/internal/cloudinfo/scrape.go b/internal/cloudinfo/scrape.go index 55b7c09d2..f709cfefb 100644 --- a/internal/cloudinfo/scrape.go +++ b/internal/cloudinfo/scrape.go @@ -16,7 +16,9 @@ package cloudinfo import ( "context" + "encoding/json" "fmt" + "os" "strconv" "sync" "time" @@ -30,6 +32,10 @@ import ( "github.com/banzaicloud/cloudinfo/internal/platform/log" ) +type VMInfoList struct { + products []types.VMInfo `json:"products"` +} + // scrapingManager manages data renewal for a given provider // retrieves data from the cloud provider and stores it in the store type scrapingManager struct { @@ -271,19 +277,52 @@ func (sm *scrapingManager) updateVirtualMachines(service, region string) error { return errors.NewWithDetails("VMs not yet cached", "provider", sm.provider, "service", service, "region", region) } + fmt.Println("-------------------") + + zones, _ := sm.store.GetZones(sm.provider, service, region) + fmt.Println(zones) + virtualMachines := make([]types.VMInfo, 0, len(vms)) - for _, vm := range vms { - prices, found := sm.store.GetPrice(sm.provider, region, vm.Type) + for _, zone := range zones { + var vmsInZone []types.VMInfo + for _, vm := range vms { + prices, found := sm.store.GetPrice(sm.provider, zone, vm.Type) + + if found { + if prices.OnDemandPrice > 0 { + vm.OnDemandPrice = prices.OnDemandPrice + } + } - if found { - if prices.OnDemandPrice > 0 { - vm.OnDemandPrice = prices.OnDemandPrice + if vm.OnDemandPrice != 0 { + virtualMachines = append(virtualMachines, vm) + vmsInZone = append(vmsInZone, vm) } } - if vm.OnDemandPrice != 0 { - virtualMachines = append(virtualMachines, vm) + vmsList := VMInfoList{ + products: vmsInZone, } + jsonString, err := json.Marshal(vmsList) + fmt.Println(err) + + fileName := fmt.Sprintf("%s.json", zone) + f, err := os.Create(fileName) + + if err != nil { + fmt.Println(err) + } + + defer f.Close() + + _, err2 := f.Write(jsonString) + + if err2 != nil { + fmt.Println(err2) + } + + fmt.Println("done") + } sm.store.DeleteVm(sm.provider, service, region) From 35975ecd3e7ecf33da4f4c1b6cd8820ccaf1be42 Mon Sep 17 00:00:00 2001 From: Maciek Urbanski Date: Mon, 25 Apr 2022 17:22:40 +0200 Subject: [PATCH 05/14] Format indent output --- internal/cloudinfo/scrape.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/cloudinfo/scrape.go b/internal/cloudinfo/scrape.go index f709cfefb..25c123579 100644 --- a/internal/cloudinfo/scrape.go +++ b/internal/cloudinfo/scrape.go @@ -33,7 +33,7 @@ import ( ) type VMInfoList struct { - products []types.VMInfo `json:"products"` + Products []types.VMInfo `json:"products"` } // scrapingManager manages data renewal for a given provider @@ -301,12 +301,13 @@ func (sm *scrapingManager) updateVirtualMachines(service, region string) error { } vmsList := VMInfoList{ - products: vmsInZone, + Products: vmsInZone, } - jsonString, err := json.Marshal(vmsList) + + jsonString, err := json.MarshalIndent(vmsList, "", "\t") fmt.Println(err) - fileName := fmt.Sprintf("%s.json", zone) + fileName := fmt.Sprintf("generated/%s.json", zone) f, err := os.Create(fileName) if err != nil { From 6a17e53a602411d00ba23f271149d09ea83e5a93 Mon Sep 17 00:00:00 2001 From: Anne Holler Date: Mon, 12 Sep 2022 09:29:54 -0700 Subject: [PATCH 06/14] Fix assumption that all zones have same shapes --- .../cloudinfo/providers/google/cloudinfo.go | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index 80b00d53b..c01bd2629 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -335,42 +335,44 @@ func (g *GceInfoer) GetVirtualMachines(region string) ([]types.VMInfo, error) { if err != nil { return nil, err } - err = g.computeSvc.MachineTypes.List(g.projectId, zones[0]).Pages(context.TODO(), func(allMts *compute.MachineTypeList) error { - for _, mt := range allMts.Items { - if _, ok := vmsMap[mt.Name]; !ok { - switch { - case mt.GuestCpus < 1: - // minimum 1 Gbps network performance for each virtual machine - ntwPerf = 1 - case mt.GuestCpus > 8: - // theoretical maximum of 16 Gbps for each virtual machine - ntwPerf = 16 - default: - // each vCPU has a 2 Gbps egress cap for peak performance - ntwPerf = uint(mt.GuestCpus * 2) - } - ntwMapper := newGceNetworkMapper() - ntwPerfCat, err := ntwMapper.MapNetworkPerf(fmt.Sprint(ntwPerf, " Gbit/s")) - if err != nil { - logger.Debug(emperror.Wrap(err, "failed to get network performance category").Error(), - map[string]interface{}{"instanceType": mt.Name}) - } - vmsMap[mt.Name] = types.VMInfo{ - Category: g.getCategory(mt.Name), - Type: mt.Name, - Cpus: float64(mt.GuestCpus), - Mem: float64(mt.MemoryMb) / 1024, - NtwPerf: fmt.Sprintf("%d Gbit/s", ntwPerf), - NtwPerfCat: ntwPerfCat, - Zones: zones, - Attributes: cloudinfo.Attributes(fmt.Sprint(mt.GuestCpus), fmt.Sprint(float64(mt.MemoryMb)/1024), ntwPerfCat, g.getCategory(mt.Name)), + for _, zone := range zones { + err = g.computeSvc.MachineTypes.List(g.projectId, zone).Pages(context.TODO(), func(allMts *compute.MachineTypeList) error { + for _, mt := range allMts.Items { + if _, ok := vmsMap[mt.Name]; !ok { + switch { + case mt.GuestCpus < 1: + // minimum 1 Gbps network performance for each virtual machine + ntwPerf = 1 + case mt.GuestCpus > 8: + // theoretical maximum of 16 Gbps for each virtual machine + ntwPerf = 16 + default: + // each vCPU has a 2 Gbps egress cap for peak performance + ntwPerf = uint(mt.GuestCpus * 2) + } + ntwMapper := newGceNetworkMapper() + ntwPerfCat, err := ntwMapper.MapNetworkPerf(fmt.Sprint(ntwPerf, " Gbit/s")) + if err != nil { + logger.Debug(emperror.Wrap(err, "failed to get network performance category").Error(), + map[string]interface{}{"instanceType": mt.Name}) + } + vmsMap[mt.Name] = types.VMInfo{ + Category: g.getCategory(mt.Name), + Type: mt.Name, + Cpus: float64(mt.GuestCpus), + Mem: float64(mt.MemoryMb) / 1024, + NtwPerf: fmt.Sprintf("%d Gbit/s", ntwPerf), + NtwPerfCat: ntwPerfCat, + Zones: zones, + Attributes: cloudinfo.Attributes(fmt.Sprint(mt.GuestCpus), fmt.Sprint(float64(mt.MemoryMb)/1024), ntwPerfCat, g.getCategory(mt.Name)), + } } } + return nil + }) + if err != nil { + return nil, err } - return nil - }) - if err != nil { - return nil, err } var vms []types.VMInfo for _, vm := range vmsMap { From 933c84af9b085a486c6b313989d01680eb63172c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Urba=C5=84ski?= Date: Thu, 18 Jul 2024 21:18:42 +0200 Subject: [PATCH 07/14] Fix saving google spot prices --- internal/cloudinfo/scrape.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/cloudinfo/scrape.go b/internal/cloudinfo/scrape.go index 25c123579..fc7998425 100644 --- a/internal/cloudinfo/scrape.go +++ b/internal/cloudinfo/scrape.go @@ -294,7 +294,16 @@ func (sm *scrapingManager) updateVirtualMachines(service, region string) error { } } - if vm.OnDemandPrice != 0 { + zonePrice := []types.ZonePrice{} + for zone, price := range prices.SpotPrice { + zonePrice = append(zonePrice, types.ZonePrice{ + Zone: zone, + Price: price, + }) + } + vm.SpotPrice = zonePrice + + if vm.OnDemandPrice != 0 || len(vm.SpotPrice) > 0 { virtualMachines = append(virtualMachines, vm) vmsInZone = append(vmsInZone, vm) } From e5707fad134caf7d55cf811258d197e859bebcc4 Mon Sep 17 00:00:00 2001 From: Anne Holler Date: Sat, 15 Feb 2025 16:36:52 -0800 Subject: [PATCH 08/14] update w/compEngId workaround and to meet luna pricing needs --- .../cloudinfo/providers/google/cloudinfo.go | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index c01bd2629..65f47d5ac 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -203,7 +203,7 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { region := r price := pricePerRegion[region] for _, mt := range allMts.Items { - if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) { + if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) && !strings.HasSuffix(mt.Name, "-metal") { if allPrices[zone] == nil { allPrices[zone] = make(map[string]types.Price) } @@ -243,6 +243,8 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { } func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) { + logger := log.WithFields(g.log, map[string]interface{}{"service": "compute"}) + logger.Debug("getting price") svcList, err := g.cbSvc.Services.List().Fields("services/displayName", "services/name").Do() if err != nil { return nil, err @@ -254,10 +256,17 @@ func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) compEngId = svc.Name } } + // Working around "Compute Engine" not found in svcList; presumably, list needs to be fetched as multiple pages + if compEngId == "" { + compEngId = "services/6F81-5844-456A" + } price := make(map[string]map[string]map[string]float64) err = g.cbSvc.Services.Skus.List(compEngId).Pages(context.Background(), func(response *cloudbilling.ListSkusResponse) error { for _, sku := range response.Skus { + if strings.Contains(sku.Description, "Upgrade Premium") || strings.Contains(sku.Description, "DWS Defined Duration") { + continue + } if sku.Category.ResourceGroup == "G1Small" || sku.Category.ResourceGroup == "F1Micro" { priceInUsd, err := g.priceInUsd(sku.PricingInfo) if err != nil { @@ -274,25 +283,26 @@ func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) price[region]["f1-micro"] = g.priceFromSku(price, region, "f1-micro", sku.Category.UsageType, priceInUsd) } } - } - if sku.Category.ResourceGroup == "N1Standard" { - if !strings.Contains(sku.Description, "Upgrade Premium") { - priceInUsd, err := g.priceInUsd(sku.PricingInfo) - if err != nil { - return err - } + } else if sku.Category.ResourceGroup == "N1Standard" { + priceInUsd, err := g.priceInUsd(sku.PricingInfo) + if err != nil { + return err + } - for _, region := range sku.ServiceRegions { - if price[region] == nil { - price[region] = make(map[string]map[string]float64) - } - if strings.Contains(sku.Description, "Instance Ram") { - price[region][types.Memory] = g.priceFromSku(price, region, types.Memory, sku.Category.UsageType, priceInUsd) - } else { - price[region][types.CPU] = g.priceFromSku(price, region, types.CPU, sku.Category.UsageType, priceInUsd) - } + for _, region := range sku.ServiceRegions { + if price[region] == nil { + price[region] = make(map[string]map[string]float64) + } + if strings.Contains(sku.Description, "Instance Ram") { + price[region][types.Memory] = g.priceFromSku(price, region, types.Memory, sku.Category.UsageType, priceInUsd) + } else if strings.Contains(sku.Description, "Instance Core") { + price[region][types.CPU] = g.priceFromSku(price, region, types.CPU, sku.Category.UsageType, priceInUsd) + } else { + logger.Debug("ignoring N1Standard", map[string]interface{}{"sku": sku}) } } + } else if sku.Category.UsageType == "OnDemand" { + logger.Debug("unrecognized sku.Category.ResourceGroup", map[string]interface{}{"sku": sku}) } } return nil From 4bbab430b97fb75eeda68dfe92c630bdab9d62d0 Mon Sep 17 00:00:00 2001 From: Anne Holler Date: Tue, 18 Feb 2025 05:47:28 -0800 Subject: [PATCH 09/14] correctly compute non-N1 shapes --- .../cloudinfo/providers/google/cloudinfo.go | 59 +++++++++++++++++-- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index 65f47d5ac..ff9089915 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -47,6 +47,7 @@ var regionNames = map[string]string{ "asia-south2": "Asia Pacific (Delhi)", "asia-southeast1": "Asia Pacific (Singapore)", "asia-southeast2": "Asia Pacific (Jakarta)", + "australia-southeast1": "Asia Pacific (Sydney)", "australia-southeast2": "Asia Pacific (Melbourne)", "europe-north1": "EU (Finland)", @@ -203,17 +204,28 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { region := r price := pricePerRegion[region] for _, mt := range allMts.Items { - if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) && !strings.HasSuffix(mt.Name, "-metal") { + if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) && !strings.HasSuffix(mt.Name, "-metal") && + !strings.HasPrefix(mt.Name, "m2-") && !strings.HasPrefix(mt.Name, "ct") { if allPrices[zone] == nil { allPrices[zone] = make(map[string]types.Price) } prices := allPrices[zone][mt.Name] - if mt.Name == "f1-micro" || mt.Name == "g1-small" { + nameSplit := strings.Split(mt.Name, "-") + family := nameSplit[0] + switch { + case mt.Name == "f1-micro" || mt.Name == "g1-small": prices.OnDemandPrice = price[mt.Name]["OnDemand"] - } else { + case family == "n1" || family == "c2": prices.OnDemandPrice = price[types.CPU]["OnDemand"]*float64(mt.GuestCpus) + price[types.Memory]["OnDemand"]*float64(mt.MemoryMb)/1024 + case family == "m1": + prices.OnDemandPrice = price["m3-cpu"]["OnDemand"]*float64(mt.GuestCpus) + price["m3-memory"]["OnDemand"]*float64(mt.MemoryMb)/1024 + case isSupportedFamily(family): + prices.OnDemandPrice = price[family+"-cpu"]["OnDemand"]*float64(mt.GuestCpus) + price[family+"-memory"]["OnDemand"]*float64(mt.MemoryMb)/1024 + default: + g.log.Warn("could not get price", map[string]interface{}{"machineTypeName": mt.Name}) } + // TODO: update this code to make it zone-friendly and ordered spotPrice := make(types.SpotPriceInfo) for _, z := range zonesInRegions[region] { if mt.Name == "f1-micro" || mt.Name == "g1-small" { @@ -242,6 +254,12 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { return allPrices, nil } +func isSupportedFamily(family string) bool { + return family == "a2" || family == "a3" || family == "c3" || family == "c3d" || family == "c4" || + family == "e2" || family == "g2" || family == "h3" || family == "n2" || family == "n4" || family == "m3" || + family == "c4a" || family == "t2a" || family == "n2d" || family == "c2d" || family == "t2d" || family == "z3" +} + func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) { logger := log.WithFields(g.log, map[string]interface{}{"service": "compute"}) logger.Debug("getting price") @@ -301,8 +319,39 @@ func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) logger.Debug("ignoring N1Standard", map[string]interface{}{"sku": sku}) } } - } else if sku.Category.UsageType == "OnDemand" { - logger.Debug("unrecognized sku.Category.ResourceGroup", map[string]interface{}{"sku": sku}) + } else if (sku.Category.UsageType == "OnDemand" || sku.Category.UsageType == "Preemptible") && + (sku.Category.ResourceGroup == "RAM" || sku.Category.ResourceGroup == "CPU") { + descSplit := strings.Split(sku.Description, " ") + if len(descSplit) < 4 { + continue + } + family := strings.ToLower(descSplit[0]) + if !isSupportedFamily(family) { + continue + } + resMatch := (descSplit[1] == "Instance" && (descSplit[2] == "Ram" || descSplit[2] == "Core")) || + (descSplit[2] == "Instance" && (descSplit[3] == "Ram" || descSplit[3] == "Core") && + (descSplit[1] == "Memory-optimized" || descSplit[1] == "Arm" || descSplit[1] == "AMD")) + if !resMatch { + continue + } + priceInUsd, err := g.priceInUsd(sku.PricingInfo) + if err != nil { + return err + } + + for _, region := range sku.ServiceRegions { + if price[region] == nil { + price[region] = make(map[string]map[string]float64) + } + if sku.Category.ResourceGroup == "RAM" { + memoryType := family + "-memory" + price[region][memoryType] = g.priceFromSku(price, region, memoryType, sku.Category.UsageType, priceInUsd) + } else { // sku.Category.ResourceGroup == "CPU" + cpuType := family + "-cpu" + price[region][cpuType] = g.priceFromSku(price, region, cpuType, sku.Category.UsageType, priceInUsd) + } + } } } return nil From 23d98fbc2ca054d48b669340af55ac99ee965617 Mon Sep 17 00:00:00 2001 From: Selvi Kadirvel Date: Sun, 24 Aug 2025 16:51:45 -0700 Subject: [PATCH 10/14] Include handling of families c4d and m4 for GCP --- .../cloudinfo/providers/google/cloudinfo.go | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index ff9089915..4cca9d86e 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -38,15 +38,15 @@ import ( const svcGke = "gke" var regionNames = map[string]string{ - "asia-east1": "Asia Pacific (Taiwan)", - "asia-east2": "Asia Pacific (Hong Kong)", - "asia-northeast1": "Asia Pacific (Tokyo)", - "asia-northeast2": "Asia Pacific (Osaka)", - "asia-northeast3": "Asia Pacific (Seoul)", - "asia-south1": "Asia Pacific (Mumbai)", - "asia-south2": "Asia Pacific (Delhi)", - "asia-southeast1": "Asia Pacific (Singapore)", - "asia-southeast2": "Asia Pacific (Jakarta)", + "asia-east1": "Asia Pacific (Taiwan)", + "asia-east2": "Asia Pacific (Hong Kong)", + "asia-northeast1": "Asia Pacific (Tokyo)", + "asia-northeast2": "Asia Pacific (Osaka)", + "asia-northeast3": "Asia Pacific (Seoul)", + "asia-south1": "Asia Pacific (Mumbai)", + "asia-south2": "Asia Pacific (Delhi)", + "asia-southeast1": "Asia Pacific (Singapore)", + "asia-southeast2": "Asia Pacific (Jakarta)", "australia-southeast1": "Asia Pacific (Sydney)", "australia-southeast2": "Asia Pacific (Melbourne)", @@ -257,7 +257,8 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { func isSupportedFamily(family string) bool { return family == "a2" || family == "a3" || family == "c3" || family == "c3d" || family == "c4" || family == "e2" || family == "g2" || family == "h3" || family == "n2" || family == "n4" || family == "m3" || - family == "c4a" || family == "t2a" || family == "n2d" || family == "c2d" || family == "t2d" || family == "z3" + family == "c4a" || family == "t2a" || family == "n2d" || family == "c2d" || family == "t2d" || family == "z3" || + family == "c4d" || family == "m4" } func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) { @@ -331,7 +332,7 @@ func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) } resMatch := (descSplit[1] == "Instance" && (descSplit[2] == "Ram" || descSplit[2] == "Core")) || (descSplit[2] == "Instance" && (descSplit[3] == "Ram" || descSplit[3] == "Core") && - (descSplit[1] == "Memory-optimized" || descSplit[1] == "Arm" || descSplit[1] == "AMD")) + (descSplit[1] == "Memory-optimized" || descSplit[1] == "Arm" || descSplit[1] == "AMD")) if !resMatch { continue } From fcedc6c2e60150d108892a2dcd4047d3db2efe2a Mon Sep 17 00:00:00 2001 From: Selvi Kadirvel Date: Sun, 24 Aug 2025 17:12:09 -0700 Subject: [PATCH 11/14] Include family a4 in GCP --- internal/cloudinfo/providers/google/cloudinfo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index 4cca9d86e..e621e53ff 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -258,7 +258,7 @@ func isSupportedFamily(family string) bool { return family == "a2" || family == "a3" || family == "c3" || family == "c3d" || family == "c4" || family == "e2" || family == "g2" || family == "h3" || family == "n2" || family == "n4" || family == "m3" || family == "c4a" || family == "t2a" || family == "n2d" || family == "c2d" || family == "t2d" || family == "z3" || - family == "c4d" || family == "m4" + family == "c4d" || family == "m4" || family == "a4" } func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) { From aa95a6656c6ded997007d788afbfb69f68dd2ef2 Mon Sep 17 00:00:00 2001 From: Selvi Kadirvel Date: Sun, 24 Aug 2025 17:56:16 -0700 Subject: [PATCH 12/14] Update github workflow action from v2 to v3 --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3db35d155..e701f79ec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,7 +29,7 @@ jobs: - name: Cache Go module dependencies id: cache-go-module-dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: go-mod-cache-${{ runner.os }}-${{ env.GO_VERSION }}-${{ hashFiles('go.sum') }} @@ -44,7 +44,7 @@ jobs: - name: Cache licenses id: cache-licenses - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .licensei.cache key: licensei-v1-${{ steps.set-git-refname.outputs.git_refname }}-${{ hashFiles('go.sum') }} @@ -89,7 +89,7 @@ jobs: - name: Cache Go module dependencies id: cache-go-module-dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: go-mod-cache-${{ runner.os }}-${{ env.GO_VERSION }}-${{ hashFiles('go.sum') }} @@ -116,7 +116,7 @@ jobs: - name: Restore Go module dependencies cache id: cache-go-module-dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: go-mod-cache-${{ runner.os }}-${{ env.GO_VERSION }}-${{ hashFiles('go.sum') }} @@ -143,7 +143,7 @@ jobs: - name: Restore Go module dependencies cache id: cache-go-module-dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: go-mod-cache-${{ runner.os }}-${{ env.GO_VERSION }}-${{ hashFiles('go.sum') }} From 50a0323d65e6bb1cdfdf542f299a57bf51fa0157 Mon Sep 17 00:00:00 2001 From: Selvi Kadirvel Date: Mon, 25 Aug 2025 12:27:49 -0700 Subject: [PATCH 13/14] Skipping instance types with unavailable on-demand prices These were the instance types were skipped when the tool was run: a4-highgpu c4-highcpu, c4-highmem, c4-standard n4-highcpu, n4-highmem, n4-standard --- internal/cloudinfo/providers/google/cloudinfo.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index e621e53ff..2f2cf40e8 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -222,6 +222,10 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { prices.OnDemandPrice = price["m3-cpu"]["OnDemand"]*float64(mt.GuestCpus) + price["m3-memory"]["OnDemand"]*float64(mt.MemoryMb)/1024 case isSupportedFamily(family): prices.OnDemandPrice = price[family+"-cpu"]["OnDemand"]*float64(mt.GuestCpus) + price[family+"-memory"]["OnDemand"]*float64(mt.MemoryMb)/1024 + if prices.OnDemandPrice == 0 { + g.log.Error("On Demand price of 0 detected indicating that price wasn't available, Skipping type (mt.name): " + mt.Name) + continue + } default: g.log.Warn("could not get price", map[string]interface{}{"machineTypeName": mt.Name}) } From c9fb7762ae910290a6717dc7560a6b322aab5d5d Mon Sep 17 00:00:00 2001 From: Anne Holler Date: Sat, 28 Feb 2026 16:03:17 -0800 Subject: [PATCH 14/14] update for new families and regions --- .../cloudinfo/providers/google/cloudinfo.go | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index 2f2cf40e8..ff01b723b 100755 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -38,6 +38,7 @@ import ( const svcGke = "gke" var regionNames = map[string]string{ + "africa-south1": "Africa (Johannesburg)", "asia-east1": "Asia Pacific (Taiwan)", "asia-east2": "Asia Pacific (Hong Kong)", "asia-northeast1": "Asia Pacific (Tokyo)", @@ -47,23 +48,36 @@ var regionNames = map[string]string{ "asia-south2": "Asia Pacific (Delhi)", "asia-southeast1": "Asia Pacific (Singapore)", "asia-southeast2": "Asia Pacific (Jakarta)", + "asia-southeast3": "Asia Pacific (Bangkok)", "australia-southeast1": "Asia Pacific (Sydney)", "australia-southeast2": "Asia Pacific (Melbourne)", - "europe-north1": "EU (Finland)", "europe-central2": "EU (Warsaw)", + "europe-north1": "EU (Finland)", + "europe-north2": "EU (Sweden)", + "europe-southwest1": "EU (Madrid)", "europe-west1": "EU (Belgium)", + "europe-west10": "EU (Berlin)", + "europe-west12": "EU (Turin)", "europe-west2": "EU (London)", "europe-west3": "EU (Frankfurt)", "europe-west4": "EU (Netherlands)", "europe-west6": "EU (Zurich)", + "europe-west8": "EU (Milan)", + "europe-west9": "EU (Paris)", + "me-central1": "Middle East (Doha)", +// "me-central2": "Middle East (Dammam)", *permission issue* + "me-west1": "Middle East (Tel Aviv)", "northamerica-northeast1": "Canada (Montréal)", "northamerica-northeast2": "Canada (Toronto)", + "northamerica-south1": "Mexico (Queretaro)", "southamerica-east1": "South America (São Paulo)", "southamerica-west1": "South America (Santiago)", "us-central1": "US Central (Iowa)", "us-east1": "US East (South Carolina)", "us-east4": "US East (Northern Virginia)", + "us-east5": "US East (Columbus, Ohio)", + "us-south1": "US West (Dallas, TX)", "us-west1": "US West (Oregon)", "us-west2": "US West (Los Angeles)", "us-west3": "US West (Salt Lake City)", @@ -205,7 +219,7 @@ func (g *GceInfoer) Initialize() (map[string]map[string]types.Price, error) { price := pricePerRegion[region] for _, mt := range allMts.Items { if !cloudinfo.Contains(unsupportedInstanceTypes, mt.Name) && !strings.HasSuffix(mt.Name, "-metal") && - !strings.HasPrefix(mt.Name, "m2-") && !strings.HasPrefix(mt.Name, "ct") { + !strings.HasPrefix(mt.Name, "m2-") && !strings.HasPrefix(mt.Name, "ct") && !strings.HasPrefix(mt.Name, "tpu") { if allPrices[zone] == nil { allPrices[zone] = make(map[string]types.Price) } @@ -262,7 +276,8 @@ func isSupportedFamily(family string) bool { return family == "a2" || family == "a3" || family == "c3" || family == "c3d" || family == "c4" || family == "e2" || family == "g2" || family == "h3" || family == "n2" || family == "n4" || family == "m3" || family == "c4a" || family == "t2a" || family == "n2d" || family == "c2d" || family == "t2d" || family == "z3" || - family == "c4d" || family == "m4" || family == "a4" + family == "c4d" || family == "m4" || family == "a4" || family == "n4a" || family == "n4d" || + family == "g4" || family == "a4x" || family == "h4d" } func (g *GceInfoer) getPrice() (map[string]map[string]map[string]float64, error) {