From 27650ca939d0c10bc684f734269e95475a269adf Mon Sep 17 00:00:00 2001
From: Emilio Garcia <egarcia@redhat.com>
Date: Fri, 18 Oct 2019 14:40:33 -0400
Subject: [PATCH 1/2] Create Bootstrap Ignition Shim Asset

---
 pkg/asset/ignition/bootstrap/bootstrap.go     |  1 -
 .../ignition/bootstrap/bootstrap_shim.go      | 85 +++++++++++++++++++
 pkg/asset/ignition/machine/master.go          |  3 +-
 pkg/asset/ignition/machine/node.go            | 59 -------------
 pkg/asset/ignition/machine/worker.go          |  3 +-
 pkg/asset/ignition/shim.go                    | 72 ++++++++++++++++
 6 files changed, 161 insertions(+), 62 deletions(-)
 create mode 100644 pkg/asset/ignition/bootstrap/bootstrap_shim.go
 delete mode 100644 pkg/asset/ignition/machine/node.go
 create mode 100644 pkg/asset/ignition/shim.go

diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go
index 80f44090220..032b46ff448 100644
--- a/pkg/asset/ignition/bootstrap/bootstrap.go
+++ b/pkg/asset/ignition/bootstrap/bootstrap.go
@@ -50,7 +50,6 @@ type bootstrapTemplateData struct {
 	BootImage             string
 }
 
-// Bootstrap is an asset that generates the ignition config for bootstrap nodes.
 type Bootstrap struct {
 	Config *igntypes.Config
 	File   *asset.File
diff --git a/pkg/asset/ignition/bootstrap/bootstrap_shim.go b/pkg/asset/ignition/bootstrap/bootstrap_shim.go
new file mode 100644
index 00000000000..9b8499da9e1
--- /dev/null
+++ b/pkg/asset/ignition/bootstrap/bootstrap_shim.go
@@ -0,0 +1,85 @@
+package bootstrap
+
+import (
+	"encoding/json"
+	"os"
+
+	igntypes "github.com/coreos/ignition/config/v2_2/types"
+	"github.com/pkg/errors"
+
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/ignition"
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/tls"
+)
+
+const (
+	bootstrapShimIgnFilename = "bootstrap_shim.ign"
+)
+
+// Shim holds data with the contents and path to a bootstrap ignition shim file
+type Shim struct {
+	Config *igntypes.Config
+	File   *asset.File
+}
+
+var _ asset.WritableAsset = (*Shim)(nil)
+
+// Dependencies returns the assets on which the Bootstrap asset depends.
+func (a *Shim) Dependencies() []asset.Asset {
+	return []asset.Asset{
+		&installconfig.InstallConfig{},
+		&tls.RootCA{},
+	}
+}
+
+// Generate generates the ignition config for the Bootstrap asset.
+func (a *Shim) Generate(dependencies asset.Parents) error {
+	installConfig := &installconfig.InstallConfig{}
+	rootCA := &tls.RootCA{}
+	dependencies.Get(installConfig, rootCA)
+
+	a.Config = ignition.PointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "bootstrap")
+	data, err := json.Marshal(a.Config)
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal Ignition config")
+	}
+	a.File = &asset.File{
+		Filename: bootstrapShimIgnFilename,
+		Data:     data,
+	}
+
+	return nil
+}
+
+// Name returns the human-friendly name of the asset.
+func (a *Shim) Name() string {
+	return "Bootstrap Shim Ignition Config"
+}
+
+// Files returns the files generated by the asset.
+func (a *Shim) Files() []*asset.File {
+	if a.File != nil {
+		return []*asset.File{a.File}
+	}
+	return []*asset.File{}
+}
+
+// Load returns the master ignitions from disk.
+func (a *Shim) Load(f asset.FileFetcher) (found bool, err error) {
+	file, err := f.FetchByName(bootstrapShimIgnFilename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	config := &igntypes.Config{}
+	if err := json.Unmarshal(file.Data, config); err != nil {
+		return false, errors.Wrapf(err, "failed to unmarshal %s", bootstrapShimIgnFilename)
+	}
+
+	a.File, a.Config = file, config
+	return true, nil
+}
diff --git a/pkg/asset/ignition/machine/master.go b/pkg/asset/ignition/machine/master.go
index bf7e860ef36..6bbf9069f4d 100644
--- a/pkg/asset/ignition/machine/master.go
+++ b/pkg/asset/ignition/machine/master.go
@@ -8,6 +8,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/ignition"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/tls"
 )
@@ -38,7 +39,7 @@ func (a *Master) Generate(dependencies asset.Parents) error {
 	rootCA := &tls.RootCA{}
 	dependencies.Get(installConfig, rootCA)
 
-	a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "master")
+	a.Config = ignition.PointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "master")
 
 	data, err := json.Marshal(a.Config)
 	if err != nil {
diff --git a/pkg/asset/ignition/machine/node.go b/pkg/asset/ignition/machine/node.go
deleted file mode 100644
index 8d46fa0b608..00000000000
--- a/pkg/asset/ignition/machine/node.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package machine
-
-import (
-	"fmt"
-	"net/url"
-
-	ignition "github.com/coreos/ignition/config/v2_2/types"
-	"github.com/vincent-petithory/dataurl"
-
-	"github.com/openshift/installer/pkg/types"
-	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
-	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
-	openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults"
-)
-
-// pointerIgnitionConfig generates a config which references the remote config
-// served by the machine config server.
-func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string) *ignition.Config {
-	var ignitionHost string
-	switch installConfig.Platform.Name() {
-	case baremetaltypes.Name:
-		// Baremetal needs to point directly at the VIP because we don't have a
-		// way to configure DNS before Ignition runs.
-		ignitionHost = fmt.Sprintf("%s:22623", installConfig.BareMetal.APIVIP)
-	case openstacktypes.Name:
-		apiVIP, err := openstackdefaults.APIVIP(installConfig.Networking)
-		if err == nil {
-			ignitionHost = fmt.Sprintf("%s:22623", apiVIP.String())
-		} else {
-			ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
-		}
-	default:
-		ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
-	}
-
-	return &ignition.Config{
-		Ignition: ignition.Ignition{
-			Version: ignition.MaxVersion.String(),
-			Config: ignition.IgnitionConfig{
-				Append: []ignition.ConfigReference{{
-					Source: func() *url.URL {
-						return &url.URL{
-							Scheme: "https",
-							Host:   ignitionHost,
-							Path:   fmt.Sprintf("/config/%s", role),
-						}
-					}().String(),
-				}},
-			},
-			Security: ignition.Security{
-				TLS: ignition.TLS{
-					CertificateAuthorities: []ignition.CaReference{{
-						Source: dataurl.EncodeBytes(rootCA),
-					}},
-				},
-			},
-		},
-	}
-}
diff --git a/pkg/asset/ignition/machine/worker.go b/pkg/asset/ignition/machine/worker.go
index e47f28fdcc5..531a700b431 100644
--- a/pkg/asset/ignition/machine/worker.go
+++ b/pkg/asset/ignition/machine/worker.go
@@ -8,6 +8,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/ignition"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/tls"
 )
@@ -38,7 +39,7 @@ func (a *Worker) Generate(dependencies asset.Parents) error {
 	rootCA := &tls.RootCA{}
 	dependencies.Get(installConfig, rootCA)
 
-	a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "worker")
+	a.Config = ignition.PointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "worker")
 
 	data, err := json.Marshal(a.Config)
 	if err != nil {
diff --git a/pkg/asset/ignition/shim.go b/pkg/asset/ignition/shim.go
new file mode 100644
index 00000000000..20f07a082cc
--- /dev/null
+++ b/pkg/asset/ignition/shim.go
@@ -0,0 +1,72 @@
+package ignition
+
+import (
+	"fmt"
+	"net/url"
+
+	ignition "github.com/coreos/ignition/config/v2_2/types"
+	"github.com/vincent-petithory/dataurl"
+
+	"github.com/openshift/installer/pkg/types"
+	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
+	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
+	openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults"
+)
+
+// PointerIgnitionConfig generates a config which references the remote config
+// served by the machine config server.
+func PointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string) *ignition.Config {
+	var ignitionHost string
+	CAReferences := []ignition.CaReference{}
+
+	// TODO(egarcia): Move this logic to the master/worker/bootstrap ignition config generation code
+	// and add parameters to service it
+	if role == "bootstrap" {
+		CAReferences = append(CAReferences, ignition.CaReference{
+			Source: installConfig.AdditionalTrustBundle,
+		})
+	} else {
+		CAReferences = append(CAReferences, ignition.CaReference{
+			Source: dataurl.EncodeBytes(rootCA),
+		})
+
+		switch installConfig.Platform.Name() {
+		case baremetaltypes.Name:
+			// Baremetal needs to point directly at the VIP because we don't have a
+			// way to configure DNS before Ignition runs.
+			ignitionHost = fmt.Sprintf("%s:22623", installConfig.BareMetal.APIVIP)
+		case openstacktypes.Name:
+			apiVIP, err := openstackdefaults.APIVIP(installConfig.Networking)
+			if err == nil {
+				ignitionHost = fmt.Sprintf("%s:22623", apiVIP.String())
+			} else {
+				ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
+			}
+		default:
+			ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
+		}
+	}
+
+	return &ignition.Config{
+		Ignition: ignition.Ignition{
+			Version: ignition.MaxVersion.String(),
+			Config: ignition.IgnitionConfig{
+				Append: []ignition.ConfigReference{{
+					Source: func() *url.URL {
+						return &url.URL{
+							Scheme: "https",
+							Host:   ignitionHost,
+							Path:   fmt.Sprintf("/config/%s", role),
+						}
+					}().String(),
+				}},
+			},
+			Security: ignition.Security{
+				TLS: ignition.TLS{
+					CertificateAuthorities: CAReferences,
+				},
+			},
+		},
+	}
+
+}

From 5411886fb81e258cc3ceedd945647c91e2605123 Mon Sep 17 00:00:00 2001
From: Emilio Garcia <egarcia@redhat.com>
Date: Mon, 21 Oct 2019 16:56:08 -0400
Subject: [PATCH 2/2] Wire Up bootstrap ignition shim asset for OpenStack
 platform

---
 data/data/config.tf                        |  9 ++++++
 data/data/openstack/bootstrap/main.tf      |  4 +++
 data/data/openstack/bootstrap/variables.tf |  5 +++
 data/data/openstack/main.tf                |  1 +
 pkg/asset/cluster/tfvars.go                |  6 +++-
 pkg/asset/ignition/bootstrap/bootstrap.go  |  1 +
 pkg/asset/ignition/shim.go                 | 37 ++++++++++++----------
 pkg/tfvars/tfvars.go                       | 22 +++++++------
 8 files changed, 58 insertions(+), 27 deletions(-)

diff --git a/data/data/config.tf b/data/data/config.tf
index 853137ad27a..9955fa0277e 100644
--- a/data/data/config.tf
+++ b/data/data/config.tf
@@ -71,6 +71,15 @@ EOF
 
 }
 
+variable "ignition_bootstrap_shim" {
+  type    = string
+  default = ""
+
+  description = <<EOF
+(internal) Ignition config file contents. This is autmoatically generated by the installer.
+EOF
+}
+
 // This variable is generated by OpenShift internally. Do not modify
 variable "cluster_id" {
   type = string
diff --git a/data/data/openstack/bootstrap/main.tf b/data/data/openstack/bootstrap/main.tf
index 3b737324862..a61045eed30 100644
--- a/data/data/openstack/bootstrap/main.tf
+++ b/data/data/openstack/bootstrap/main.tf
@@ -63,6 +63,10 @@ resource "openstack_objectstorage_object_v1" "ignition" {
 }
 
 data "ignition_config" "redirect" {
+  append {
+    source = "data:text/plain;charset=utf-8;base64,${base64encode(var.ignition_shim)}"
+  }
+
   append {
     source = "${var.swift_url}/${openstack_objectstorage_container_v1.container.name}/${random_password.random.result}"
   }
diff --git a/data/data/openstack/bootstrap/variables.tf b/data/data/openstack/bootstrap/variables.tf
index 04a8c4d9554..a1516dcdc75 100644
--- a/data/data/openstack/bootstrap/variables.tf
+++ b/data/data/openstack/bootstrap/variables.tf
@@ -25,6 +25,11 @@ variable "ignition" {
   description = "The content of the bootstrap ignition file."
 }
 
+variable "ignition_shim" {
+  type = string
+  description = "A pointer to the bootstrap ignition file."
+}
+
 variable "flavor_name" {
   type = string
   description = "The Nova flavor for the bootstrap node."
diff --git a/data/data/openstack/main.tf b/data/data/openstack/main.tf
index 97e49e1d6aa..996aa5a0684 100644
--- a/data/data/openstack/main.tf
+++ b/data/data/openstack/main.tf
@@ -30,6 +30,7 @@ module "bootstrap" {
   base_image_id      = data.openstack_images_image_v2.base_image.id
   flavor_name        = var.openstack_master_flavor_name
   ignition           = var.ignition_bootstrap
+  ignition_shim      = var.ignition_bootstrap_shim
   api_int_ip         = var.openstack_api_int_ip
   node_dns_ip        = var.openstack_node_dns_ip
   external_network   = var.openstack_external_network
diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go
index 231cfb2d44d..6e63f9abf47 100644
--- a/pkg/asset/cluster/tfvars.go
+++ b/pkg/asset/cluster/tfvars.go
@@ -77,6 +77,7 @@ func (t *TerraformVariables) Dependencies() []asset.Asset {
 		new(rhcos.Image),
 		new(rhcos.BootstrapImage),
 		&bootstrap.Bootstrap{},
+		&bootstrap.Shim{},
 		&machine.Master{},
 		&machines.Master{},
 		&machines.Worker{},
@@ -89,12 +90,13 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 	clusterID := &installconfig.ClusterID{}
 	installConfig := &installconfig.InstallConfig{}
 	bootstrapIgnAsset := &bootstrap.Bootstrap{}
+	bootstrapIgnShimAsset := &bootstrap.Shim{}
 	masterIgnAsset := &machine.Master{}
 	mastersAsset := &machines.Master{}
 	workersAsset := &machines.Worker{}
 	rhcosImage := new(rhcos.Image)
 	rhcosBootstrapImage := new(rhcos.BootstrapImage)
-	parents.Get(clusterID, installConfig, bootstrapIgnAsset, masterIgnAsset, mastersAsset, workersAsset, rhcosImage, rhcosBootstrapImage)
+	parents.Get(clusterID, installConfig, bootstrapIgnAsset, bootstrapIgnShimAsset, masterIgnAsset, mastersAsset, workersAsset, rhcosImage, rhcosBootstrapImage)
 
 	platform := installConfig.Config.Platform.Name()
 	switch platform {
@@ -103,6 +105,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 	}
 
 	masterIgn := string(masterIgnAsset.Files()[0].Data)
+	bootstrapShim := string(bootstrapIgnShimAsset.Files()[0].Data)
 	bootstrapIgn, err := injectInstallInfo(bootstrapIgnAsset.Files()[0].Data)
 	if err != nil {
 		return errors.Wrap(err, "unable to inject installation info")
@@ -115,6 +118,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		installConfig.Config.BaseDomain,
 		&installConfig.Config.Networking.MachineCIDR.IPNet,
 		bootstrapIgn,
+		bootstrapShim,
 		masterIgn,
 		masterCount,
 	)
diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go
index 032b46ff448..80f44090220 100644
--- a/pkg/asset/ignition/bootstrap/bootstrap.go
+++ b/pkg/asset/ignition/bootstrap/bootstrap.go
@@ -50,6 +50,7 @@ type bootstrapTemplateData struct {
 	BootImage             string
 }
 
+// Bootstrap is an asset that generates the ignition config for bootstrap nodes.
 type Bootstrap struct {
 	Config *igntypes.Config
 	File   *asset.File
diff --git a/pkg/asset/ignition/shim.go b/pkg/asset/ignition/shim.go
index 20f07a082cc..7af166a86b2 100644
--- a/pkg/asset/ignition/shim.go
+++ b/pkg/asset/ignition/shim.go
@@ -18,18 +18,21 @@ import (
 func PointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string) *ignition.Config {
 	var ignitionHost string
 	CAReferences := []ignition.CaReference{}
+	configReference := []ignition.ConfigReference{}
+
+	CAReferences = append(CAReferences, ignition.CaReference{
+		Source: dataurl.EncodeBytes(rootCA),
+	})
 
 	// TODO(egarcia): Move this logic to the master/worker/bootstrap ignition config generation code
 	// and add parameters to service it
 	if role == "bootstrap" {
-		CAReferences = append(CAReferences, ignition.CaReference{
-			Source: installConfig.AdditionalTrustBundle,
-		})
+		if installConfig.AdditionalTrustBundle != "" {
+			CAReferences = append(CAReferences, ignition.CaReference{
+				Source: installConfig.AdditionalTrustBundle,
+			})
+		}
 	} else {
-		CAReferences = append(CAReferences, ignition.CaReference{
-			Source: dataurl.EncodeBytes(rootCA),
-		})
-
 		switch installConfig.Platform.Name() {
 		case baremetaltypes.Name:
 			// Baremetal needs to point directly at the VIP because we don't have a
@@ -45,21 +48,23 @@ func PointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro
 		default:
 			ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
 		}
+
+		configReference = []ignition.ConfigReference{{
+			Source: func() *url.URL {
+				return &url.URL{
+					Scheme: "https",
+					Host:   ignitionHost,
+					Path:   fmt.Sprintf("/config/%s", role),
+				}
+			}().String(),
+		}}
 	}
 
 	return &ignition.Config{
 		Ignition: ignition.Ignition{
 			Version: ignition.MaxVersion.String(),
 			Config: ignition.IgnitionConfig{
-				Append: []ignition.ConfigReference{{
-					Source: func() *url.URL {
-						return &url.URL{
-							Scheme: "https",
-							Host:   ignitionHost,
-							Path:   fmt.Sprintf("/config/%s", role),
-						}
-					}().String(),
-				}},
+				Append: configReference,
 			},
 			Security: ignition.Security{
 				TLS: ignition.TLS{
diff --git a/pkg/tfvars/tfvars.go b/pkg/tfvars/tfvars.go
index 4e772934e80..71f39980546 100644
--- a/pkg/tfvars/tfvars.go
+++ b/pkg/tfvars/tfvars.go
@@ -14,20 +14,22 @@ type config struct {
 	MachineCIDR   string `json:"machine_cidr"`
 	Masters       int    `json:"master_count,omitempty"`
 
-	IgnitionBootstrap string `json:"ignition_bootstrap,omitempty"`
-	IgnitionMaster    string `json:"ignition_master,omitempty"`
+	IgnitionBootstrap     string `json:"ignition_bootstrap,omitempty"`
+	IgnitionBootstrapShim string `json:"ignition_bootstrap_shim,omitempty"`
+	IgnitionMaster        string `json:"ignition_master,omitempty"`
 }
 
 // TFVars generates terraform.tfvar JSON for launching the cluster.
-func TFVars(clusterID string, clusterDomain string, baseDomain string, machineCIDR *net.IPNet, bootstrapIgn string, masterIgn string, masterCount int) ([]byte, error) {
+func TFVars(clusterID string, clusterDomain string, baseDomain string, machineCIDR *net.IPNet, bootstrapIgn string, bootstrapIgnShim string, masterIgn string, masterCount int) ([]byte, error) {
 	config := &config{
-		ClusterID:         clusterID,
-		ClusterDomain:     strings.TrimSuffix(clusterDomain, "."),
-		BaseDomain:        strings.TrimSuffix(baseDomain, "."),
-		MachineCIDR:       machineCIDR.String(),
-		Masters:           masterCount,
-		IgnitionBootstrap: bootstrapIgn,
-		IgnitionMaster:    masterIgn,
+		ClusterID:             clusterID,
+		ClusterDomain:         strings.TrimSuffix(clusterDomain, "."),
+		BaseDomain:            strings.TrimSuffix(baseDomain, "."),
+		MachineCIDR:           machineCIDR.String(),
+		Masters:               masterCount,
+		IgnitionBootstrap:     bootstrapIgn,
+		IgnitionBootstrapShim: bootstrapIgnShim,
+		IgnitionMaster:        masterIgn,
 	}
 
 	return json.MarshalIndent(config, "", "  ")