From 71374795367bc46feddf51787a99a29eed424b01 Mon Sep 17 00:00:00 2001 From: Emerson Dove <52636744+EmersonDove@users.noreply.github.com> Date: Tue, 19 Jul 2022 10:35:15 -0400 Subject: [PATCH 1/6] Add GCP integration --- fogros2/fogros2/__init__.py | 1 + fogros2/fogros2/cloud_instance.py | 15 +-- fogros2/fogros2/dds_config_builder.py | 4 +- fogros2/fogros2/gcp_cloud_instance.py | 139 ++++++++++++++++++++++++++ fogros2/fogros2/scp.py | 9 +- 5 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 fogros2/fogros2/gcp_cloud_instance.py diff --git a/fogros2/fogros2/__init__.py b/fogros2/fogros2/__init__.py index af6f673..eae3577 100755 --- a/fogros2/fogros2/__init__.py +++ b/fogros2/fogros2/__init__.py @@ -32,5 +32,6 @@ # MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. from .aws_cloud_instance import AWSCloudInstance # noqa: F401 +from .gcp_cloud_instance import GCPCloudInstance from .cloud_node import CloudNode # noqa: F401 from .launch_description import FogROSLaunchDescription # noqa: F401 diff --git a/fogros2/fogros2/cloud_instance.py b/fogros2/fogros2/cloud_instance.py index 7bc9743..fb01827 100644 --- a/fogros2/fogros2/cloud_instance.py +++ b/fogros2/fogros2/cloud_instance.py @@ -84,6 +84,7 @@ def __init__( self.cloud_service_provider = None self.dockers = [] self.launch_foxglove = launch_foxglove + self._username = 'ubuntu' @abc.abstractmethod def create(self): @@ -104,7 +105,7 @@ def info(self, flush_to_disk=True): return info_dict def connect(self): - self.scp = SCPClient(self._ip, self._ssh_key_path) + self.scp = SCPClient(self._ip, self._ssh_key_path, username=self._username) self.scp.connect() @property @@ -173,7 +174,7 @@ def configure_rosbridge(self): rosbridge_launch_script = ( "ssh -o StrictHostKeyChecking=no -i " f"{self._ssh_key_path}" - " ubuntu@" + f" {self._username}@" f"{self._ip}" f' "source /opt/ros/{self.ros_distro}/setup.bash && ' 'ros2 launch rosbridge_server rosbridge_websocket_launch.xml &"' @@ -203,8 +204,8 @@ def push_ros_workspace(self): make_zip_file(workspace_path, zip_dst) self.scp.execute_cmd("echo removing old workspace") self.scp.execute_cmd("rm -rf ros_workspace.zip ros2_ws fog_ws") - self.scp.send_file(f"{zip_dst}.zip", "/home/ubuntu/") - self.scp.execute_cmd("unzip -q /home/ubuntu/ros_workspace.zip") + self.scp.send_file(f"{zip_dst}.zip", f"/home/{self._username}/") + self.scp.execute_cmd(f"unzip -q /home/{self._username}/ros_workspace.zip") self.scp.execute_cmd("echo successfully extracted new workspace") def push_to_cloud_nodes(self): @@ -223,7 +224,7 @@ def push_and_setup_vpn(self): def configure_DDS(self): # configure DDS - self.cyclone_builder = CycloneConfigBuilder(["10.0.0.1"]) + self.cyclone_builder = CycloneConfigBuilder(["10.0.0.1"], username=self._username) self.cyclone_builder.generate_config_file() self.scp.send_file("/tmp/cyclonedds.xml", "~/cyclonedds.xml") @@ -231,9 +232,9 @@ def launch_cloud_node(self): cmd_builder = BashBuilder() cmd_builder.append(f"source /opt/ros/{self.ros_distro}/setup.bash") cmd_builder.append( - "cd /home/ubuntu/fog_ws && colcon build --cmake-clean-cache" + f"cd /home/{self._username}/fog_ws && colcon build --cmake-clean-cache" ) - cmd_builder.append(". /home/ubuntu/fog_ws/install/setup.bash") + cmd_builder.append(f". /home/{self._username}/fog_ws/install/setup.bash") cmd_builder.append(self.cyclone_builder.env_cmd) ros_domain_id = os.environ.get("ROS_DOMAIN_ID") if not ros_domain_id: diff --git a/fogros2/fogros2/dds_config_builder.py b/fogros2/fogros2/dds_config_builder.py index f88d00f..adae1e2 100644 --- a/fogros2/fogros2/dds_config_builder.py +++ b/fogros2/fogros2/dds_config_builder.py @@ -53,12 +53,12 @@ def generate_config_file(self): class CycloneConfigBuilder(DDSConfigBuilder): - def __init__(self, ip_addresses): + def __init__(self, ip_addresses, username='ubuntu'): super().__init__(ip_addresses) self.config_save_path = "/tmp/cyclonedds.xml" self.env_cmd = ( "export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && " - "export CYCLONEDDS_URI=file:///home/ubuntu/cyclonedds.xml" + f"export CYCLONEDDS_URI=file:///home/{username}/cyclonedds.xml" ) def generate_config_file(self): diff --git a/fogros2/fogros2/gcp_cloud_instance.py b/fogros2/fogros2/gcp_cloud_instance.py new file mode 100644 index 0000000..f6b8369 --- /dev/null +++ b/fogros2/fogros2/gcp_cloud_instance.py @@ -0,0 +1,139 @@ +# Copyright 2022 The Regents of the University of California (Regents) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright ©2022. The Regents of the University of California (Regents). +# All Rights Reserved. Permission to use, copy, modify, and distribute this +# software and its documentation for educational, research, and not-for-profit +# purposes, without fee and without a signed licensing agreement, is hereby +# granted, provided that the above copyright notice, this paragraph and the +# following two paragraphs appear in all copies, modifications, and +# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150 +# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201, +# otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial +# licensing opportunities. IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY +# FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, +# INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS +# DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +# DAMAGE. REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, +# PROVIDED HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +import json +import os + +import subprocess +import uuid + +from .cloud_instance import CloudInstance + + +class GCPCloudInstance(CloudInstance): + """AWS Implementation of CloudInstance.""" + + def __init__( + self, + gcp_project_id, + ami_image='projects/debian-cloud/global/images/debian-11-bullseye-v20220621', + zone="us-central1-a", + machine_type="e2-medium", + disk_size=30, + **kwargs, + ): + super().__init__(**kwargs) + self.cloud_service_provider = "GCP" + + id_ = str(uuid.uuid4())[0:8] + self._name = f'{id_}{self._name}' + + self.zone = zone + self.type = machine_type + self.compute_instance_disk_size = disk_size # GB + self.gcp_ami_image = ami_image + + self._working_dir = os.path.join(self._working_dir_base, self._name) + os.makedirs(self._working_dir, exist_ok=True) + + self._project_id = gcp_project_id + + # after config + self._ssh_key = None + + self.create() + + def create(self): + self.logger.info(f"Creating new GCP Compute Engine instance with name {self._name}") + self.create_compute_engine_instance() + self.info(flush_to_disk=True) + self.connect() + self.install_ros() + self.install_colcon() + self.install_cloud_dependencies() + self.push_ros_workspace() + self.info(flush_to_disk=True) + self._is_created = True + + def info(self, flush_to_disk=True): + info_dict = super().info(flush_to_disk) + info_dict["compute_region"] = self.zone + info_dict["compute_instance_type"] = self.type + info_dict["disk_size"] = self.compute_instance_disk_size + info_dict["compute_instance_id"] = self._name + if flush_to_disk: + with open(os.path.join(self._working_dir, "info"), "w+") as f: + json.dump(info_dict, f) + return info_dict + + def create_compute_engine_instance(self): + ip = subprocess.check_output(f'gcloud compute instances create {self._name} ' + f'--project={self._project_id} --zone={self.zone} --machine-type={self.type} ' + '--network-interface=network-tier=PREMIUM,subnet=default ' + '--maintenance-policy=MIGRATE --provisioning-model=STANDARD ' + '--scopes=https://www.googleapis.com/auth/devstorage.read_only,' + 'https://www.googleapis.com/auth/logging.write,' + 'https://www.googleapis.com/auth/monitoring.write,' + 'https://www.googleapis.com/auth/servicecontrol,' + 'https://www.googleapis.com/auth/service.management.readonly,' + 'https://www.googleapis.com/auth/trace.append ' + '--create-disk=auto-delete=yes,' + 'boot=yes,' + f'device-name={self._name},' + f'image={self.gcp_ami_image},' + 'mode=rw,' + f'size={self.compute_instance_disk_size},' + f'type=projects/{self._project_id}/zones/{self.zone}/diskTypes/pd-balanced ' + '--no-shielded-secure-boot ' + '--shielded-vtpm ' + '--shielded-integrity-monitoring ' + '--reservation-affinity=any | ' + "awk '{print $5}' | " + "sed -n 2p", shell=True).decode().strip() + + # Generate SSH keys + os.system("printf '\n\n' | gcloud compute ssh instance-1 --zone us-central1-a") + + # Username + self._username = (open('~/.ssh/google_compute_engine.pub').read()).split(' ')[-1].strip().split('@')[0] + + # Get the username: + self._ip = ip + + self._ssh_key_path = '~/.ssh/google_compute_engine' + self._is_created = True + + self.logger.info( + f"Created {self.type} instance named {self._name} " + f"with id {self._name} and public IP address {self._ip}" + ) diff --git a/fogros2/fogros2/scp.py b/fogros2/fogros2/scp.py index c74b745..5916ba5 100755 --- a/fogros2/fogros2/scp.py +++ b/fogros2/fogros2/scp.py @@ -45,12 +45,17 @@ class SCPClient: - def __init__(self, ip, ssh_key_path): + def __init__(self, ip, ssh_key_path, username=None): self.ip = ip self.ssh_key = paramiko.RSAKey.from_private_key_file(ssh_key_path) self.ssh_client = paramiko.SSHClient() self.logger = logging.get_logger(__name__) + if username is None: + self.username = 'ubuntu' + else: + self.username = username + def connect(self): self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) connected = False @@ -58,7 +63,7 @@ def connect(self): try: self.ssh_client.connect( hostname=self.ip, - username="ubuntu", + username=self.username, pkey=self.ssh_key, look_for_keys=False, ) From 3617b313bf37921be59483f0df083d0c5726e7ab Mon Sep 17 00:00:00 2001 From: Emerson Dove <52636744+EmersonDove@users.noreply.github.com> Date: Tue, 19 Jul 2022 10:55:00 -0400 Subject: [PATCH 2/6] Add talker example --- fogros2_examples/launch/talker.gcp.launch.py | 58 ++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 fogros2_examples/launch/talker.gcp.launch.py diff --git a/fogros2_examples/launch/talker.gcp.launch.py b/fogros2_examples/launch/talker.gcp.launch.py new file mode 100644 index 0000000..78719b0 --- /dev/null +++ b/fogros2_examples/launch/talker.gcp.launch.py @@ -0,0 +1,58 @@ +# Copyright 2022 The Regents of the University of California (Regents) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright ©2022. The Regents of the University of California (Regents). +# All Rights Reserved. Permission to use, copy, modify, and distribute this +# software and its documentation for educational, research, and not-for-profit +# purposes, without fee and without a signed licensing agreement, is hereby +# granted, provided that the above copyright notice, this paragraph and the +# following two paragraphs appear in all copies, modifications, and +# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150 +# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201, +# otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial +# licensing opportunities. IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY +# FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, +# INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS +# DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +# DAMAGE. REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, +# PROVIDED HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +from launch_ros.actions import Node + +import fogros2 + + +def generate_launch_description(): + """Talker example that launches the listener on AWS.""" + ld = fogros2.FogROSLaunchDescription() + machine1 = fogros2.GCPCloudInstance( + region="us-west-1", ec2_instance_type="t2.micro" + ) + + listener_node = Node( + package="fogros2_examples", executable="listener", output="screen" + ) + + talker_node = fogros2.CloudNode( + package="fogros2_examples", + executable="talker", + output="screen", + machine=machine1, + ) + ld.add_action(talker_node) + ld.add_action(listener_node) + return ld From 959b5a0079279f5be4a199e4e3d507da27a42837 Mon Sep 17 00:00:00 2001 From: Emerson Dove <52636744+EmersonDove@users.noreply.github.com> Date: Tue, 19 Jul 2022 21:21:25 +0000 Subject: [PATCH 3/6] Add compute engine integration, begin kube integration --- fogros2/fogros2/cloud_instance.py | 2 + fogros2/fogros2/gcp_cloud_instance.py | 35 ++++-- fogros2/fogros2/kubernetes/__init__.py | 0 fogros2/fogros2/kubernetes/gcp_kubernetes.py | 126 +++++++++++++++++++ fogros2/fogros2/kubernetes/pod.json | 33 +++++ fogros2/fogros2/kubernetes/ssh.yaml | 14 +++ fogros2/fogros2/kubernetes/vpn.yaml | 14 +++ fogros2_examples/launch/talker.gcp.launch.py | 2 +- 8 files changed, 212 insertions(+), 14 deletions(-) create mode 100644 fogros2/fogros2/kubernetes/__init__.py create mode 100644 fogros2/fogros2/kubernetes/gcp_kubernetes.py create mode 100644 fogros2/fogros2/kubernetes/pod.json create mode 100644 fogros2/fogros2/kubernetes/ssh.yaml create mode 100644 fogros2/fogros2/kubernetes/vpn.yaml diff --git a/fogros2/fogros2/cloud_instance.py b/fogros2/fogros2/cloud_instance.py index fb01827..35e3ebb 100644 --- a/fogros2/fogros2/cloud_instance.py +++ b/fogros2/fogros2/cloud_instance.py @@ -162,6 +162,8 @@ def install_ros(self): # install ros2 packages self.apt_install(f"ros-{self.ros_distro}-desktop") + self.apt_install('python3-colcon-common-extensions') + # source environment self.scp.execute_cmd(f"source /opt/ros/{self.ros_distro}/setup.bash") diff --git a/fogros2/fogros2/gcp_cloud_instance.py b/fogros2/fogros2/gcp_cloud_instance.py index f6b8369..deb300d 100644 --- a/fogros2/fogros2/gcp_cloud_instance.py +++ b/fogros2/fogros2/gcp_cloud_instance.py @@ -21,7 +21,7 @@ # distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150 # Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201, # otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial -# licensing opportunities. IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY +# licensing opportunities. IN NO EVEpNT SHALL REGENTS BE LIABLE TO ANY PARTY # FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, # INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS # DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH @@ -35,28 +35,29 @@ import os import subprocess +import time import uuid from .cloud_instance import CloudInstance class GCPCloudInstance(CloudInstance): - """AWS Implementation of CloudInstance.""" + """GCP Implementation of CloudInstance.""" def __init__( self, - gcp_project_id, - ami_image='projects/debian-cloud/global/images/debian-11-bullseye-v20220621', + project_id, + ami_image='projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20220712a', zone="us-central1-a", machine_type="e2-medium", - disk_size=30, + disk_size=10, **kwargs, ): super().__init__(**kwargs) self.cloud_service_provider = "GCP" id_ = str(uuid.uuid4())[0:8] - self._name = f'{id_}{self._name}' + self._name = f'fog-{id_}-{self._name}' self.zone = zone self.type = machine_type @@ -66,7 +67,7 @@ def __init__( self._working_dir = os.path.join(self._working_dir_base, self._name) os.makedirs(self._working_dir, exist_ok=True) - self._project_id = gcp_project_id + self._project_id = project_id # after config self._ssh_key = None @@ -97,6 +98,8 @@ def info(self, flush_to_disk=True): return info_dict def create_compute_engine_instance(self): + os.system(f'gcloud config set project {self._project_id}') + ip = subprocess.check_output(f'gcloud compute instances create {self._name} ' f'--project={self._project_id} --zone={self.zone} --machine-type={self.type} ' '--network-interface=network-tier=PREMIUM,subnet=default ' @@ -121,16 +124,22 @@ def create_compute_engine_instance(self): "awk '{print $5}' | " "sed -n 2p", shell=True).decode().strip() + # Verifies the response was an ip + if len(ip.split('.')) != 4: + raise Exception(f'Error creating instance: {ip}') + + self._ip = ip + # Generate SSH keys - os.system("printf '\n\n' | gcloud compute ssh instance-1 --zone us-central1-a") + os.system(f"printf '\n\n' | gcloud compute ssh {self._name} --zone {self.zone}") - # Username - self._username = (open('~/.ssh/google_compute_engine.pub').read()).split(' ')[-1].strip().split('@')[0] + user = subprocess.check_output('whoami').decode().strip() - # Get the username: - self._ip = ip + # Username + self._username = (open(f'/home/{user}/.ssh/google_compute_engine.pub'). + read()).split(' ')[-1].strip().split('@')[0] - self._ssh_key_path = '~/.ssh/google_compute_engine' + self._ssh_key_path = f'/home/{user}/.ssh/google_compute_engine' self._is_created = True self.logger.info( diff --git a/fogros2/fogros2/kubernetes/__init__.py b/fogros2/fogros2/kubernetes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fogros2/fogros2/kubernetes/gcp_kubernetes.py b/fogros2/fogros2/kubernetes/gcp_kubernetes.py new file mode 100644 index 0000000..596310a --- /dev/null +++ b/fogros2/fogros2/kubernetes/gcp_kubernetes.py @@ -0,0 +1,126 @@ +# Copyright 2022 The Regents of the University of California (Regents) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright ©2022. The Regents of the University of California (Regents). +# All Rights Reserved. Permission to use, copy, modify, and distribute this +# software and its documentation for educational, research, and not-for-profit +# purposes, without fee and without a signed licensing agreement, is hereby +# granted, provided that the above copyright notice, this paragraph and the +# following two paragraphs appear in all copies, modifications, and +# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150 +# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201, +# otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial +# licensing opportunities. IN NO EVEpNT SHALL REGENTS BE LIABLE TO ANY PARTY +# FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, +# INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS +# DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +# DAMAGE. REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, +# PROVIDED HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +import json +import os + +import subprocess +import time +import uuid + +from .cloud_instance import CloudInstance + + +class GCPCloudInstance(CloudInstance): + """Kubernetes Implementation of CloudInstance.""" + + def __init__( + self, + project_id, + container_image='ubuntu', + zone="us-central1-a", + mCpu=500, + mB=200, + **kwargs, + ): + super().__init__(**kwargs) + self.cloud_service_provider = "Kubernetes" + + id_ = str(uuid.uuid4())[0:8] + self._name = f'fog-{id_}-{self._name}' + + self.zone = zone + self.type = machine_type + self.container_image = container_image + + self._working_dir = os.path.join(self._working_dir_base, self._name) + os.makedirs(self._working_dir, exist_ok=True) + + self._project_id = project_id + + # after config + self._ssh_key = None + + self.create() + + def create(self): + self.logger.info(f"Creating new GCP Compute Engine instance with name {self._name}") + self.create_compute_engine_instance() + self.info(flush_to_disk=True) + self.connect() + self.install_ros() + self.install_colcon() + self.install_cloud_dependencies() + self.push_ros_workspace() + self.info(flush_to_disk=True) + self._is_created = True + + def info(self, flush_to_disk=True): + info_dict = super().info(flush_to_disk) + info_dict["compute_region"] = self.zone + info_dict["compute_instance_type"] = self.type + info_dict["disk_size"] = self.compute_instance_disk_size + info_dict["compute_instance_id"] = self._name + if flush_to_disk: + with open(os.path.join(self._working_dir, "info"), "w+") as f: + json.dump(info_dict, f) + return info_dict + + def create_compute_engine_instance(self): + user = subprocess.check_output('whoami').decode().strip() + + # Username + self._username = (open(f'/home/{user}/.ssh/google_compute_engine.pub'). + read()).split(' ')[-1].strip().split('@')[0] + + self._ssh_key_path = f'/home/{user}/.ssh/google_compute_engine' + self._is_created = True + + self.logger.info( + f"Created {self.type} instance named {self._name} " + f"with id {self._name} and public IP address {self._ip}" + ) + + command = 'useradd "$USERNAME" -m -s /bin/bash && ' \ + 'mkdir "/home/$USERNAME/.ssh" && ' \ + 'echo "$SSH_PUBKEY" >> "/home/$USERNAME/.ssh/authorized_keys" && ' \ + 'chmod -R u=rwX "/home/$USERNAME/.ssh" && ' \ + 'chown -R "$USERNAME:$USERNAME" "/home/$USERNAME/.ssh" && ' \ + 'echo "$USERNAME ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/shade && ' \ + 'service ssh restart && ' \ + 'wg-quick the vpn configuration && ' \ + 'echo "$VPN_CONFIG" | base64 --decode > /home/app/vpn.conf && ' \ + 'cp /home/app/vpn.conf /etc/wireguard/wg0.conf && ' \ + 'chmod 600 /etc/wireguard/wg0.conf && ' \ + 'sudo wg-quick up wg0 && ' \ + 'sleep infinity' diff --git a/fogros2/fogros2/kubernetes/pod.json b/fogros2/fogros2/kubernetes/pod.json new file mode 100644 index 0000000..347e318 --- /dev/null +++ b/fogros2/fogros2/kubernetes/pod.json @@ -0,0 +1,33 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "strategy" + }, + "spec": { + "restartPolicy": "Never", + "containers": [ + { + "name": "", + "image": "", + "imagePullPolicy": "Always", + "securityContext": { + "capabilities": { + "add": ["NET_ADMIN"] + } + }, + "resources": { + "requests":{ + "memory": "<128Mi means 128 Megabytes>", + "cpu": "<125m means 125 millicpu>" + }, + "limits": { + "memory": "<128Mi means 128 Megabytes>", + "cpu": "<125m means 125 millicpu>" + } + }, + "env": [] + } + ] + } +} diff --git a/fogros2/fogros2/kubernetes/ssh.yaml b/fogros2/fogros2/kubernetes/ssh.yaml new file mode 100644 index 0000000..25c1368 --- /dev/null +++ b/fogros2/fogros2/kubernetes/ssh.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: ssh-balancer + namespace: ros +spec: + type: LoadBalancer + ports: + - port: 22 + targetPort: 22 + name: ssh + protocol: TCP + selector: + app: ssh-pod diff --git a/fogros2/fogros2/kubernetes/vpn.yaml b/fogros2/fogros2/kubernetes/vpn.yaml new file mode 100644 index 0000000..5147cfc --- /dev/null +++ b/fogros2/fogros2/kubernetes/vpn.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: vpn-balancer + namespace: ros +spec: + type: LoadBalancer + ports: + - port: 51820 + targetPort: 51820 + name: vpn + protocol: UDP + selector: + app: ssh-pod diff --git a/fogros2_examples/launch/talker.gcp.launch.py b/fogros2_examples/launch/talker.gcp.launch.py index 78719b0..31156e8 100644 --- a/fogros2_examples/launch/talker.gcp.launch.py +++ b/fogros2_examples/launch/talker.gcp.launch.py @@ -40,7 +40,7 @@ def generate_launch_description(): """Talker example that launches the listener on AWS.""" ld = fogros2.FogROSLaunchDescription() machine1 = fogros2.GCPCloudInstance( - region="us-west-1", ec2_instance_type="t2.micro" + project_id='shade-prod' ) listener_node = Node( From a988fa4c1155c0df8a29fae59a2a29d2c8070147 Mon Sep 17 00:00:00 2001 From: Emerson Dove <52636744+EmersonDove@users.noreply.github.com> Date: Wed, 20 Jul 2022 10:38:45 -0400 Subject: [PATCH 4/6] Add kubernetes integration --- fogros2/fogros2/__init__.py | 1 + fogros2/fogros2/cloud_instance.py | 6 + fogros2/fogros2/kubernetes/gcp_kubernetes.py | 140 ++++++++++++++---- fogros2/fogros2/kubernetes/pod.json | 2 +- fogros2/fogros2/kubernetes/ssh.json | 22 +++ fogros2/fogros2/kubernetes/ssh.yaml | 14 -- fogros2/fogros2/kubernetes/vpn.json | 22 +++ fogros2/fogros2/kubernetes/vpn.yaml | 14 -- fogros2/fogros2/util.py | 24 +++ fogros2/fogros2/vpn.py | 5 +- fogros2_examples/launch/talker.gcp.launch.py | 2 +- fogros2_examples/launch/talker.kube.launch.py | 56 +++++++ 12 files changed, 246 insertions(+), 62 deletions(-) create mode 100644 fogros2/fogros2/kubernetes/ssh.json delete mode 100644 fogros2/fogros2/kubernetes/ssh.yaml create mode 100644 fogros2/fogros2/kubernetes/vpn.json delete mode 100644 fogros2/fogros2/kubernetes/vpn.yaml create mode 100644 fogros2_examples/launch/talker.kube.launch.py diff --git a/fogros2/fogros2/__init__.py b/fogros2/fogros2/__init__.py index eae3577..0855c05 100755 --- a/fogros2/fogros2/__init__.py +++ b/fogros2/fogros2/__init__.py @@ -33,5 +33,6 @@ from .aws_cloud_instance import AWSCloudInstance # noqa: F401 from .gcp_cloud_instance import GCPCloudInstance +from .kubernetes.gcp_kubernetes import GCPKubeInstance from .cloud_node import CloudNode # noqa: F401 from .launch_description import FogROSLaunchDescription # noqa: F401 diff --git a/fogros2/fogros2/cloud_instance.py b/fogros2/fogros2/cloud_instance.py index 35e3ebb..b37bd27 100644 --- a/fogros2/fogros2/cloud_instance.py +++ b/fogros2/fogros2/cloud_instance.py @@ -72,6 +72,7 @@ def __init__( self.cyclone_builder = None self.scp = None self._ip = None + self._vpn_ip = None self.ros_workspace = ros_workspace self.ros_distro = os.getenv("ROS_DISTRO") self.logger.debug(f"Using ROS workspace: {self.ros_workspace}") @@ -112,6 +113,11 @@ def connect(self): def ip(self): return self._ip + @property + def vpn_ip(self): + # Use this when the VPN IP is not None. + return self._vpn_ip + @property def is_created(self): return self._is_created diff --git a/fogros2/fogros2/kubernetes/gcp_kubernetes.py b/fogros2/fogros2/kubernetes/gcp_kubernetes.py index 596310a..bdbf508 100644 --- a/fogros2/fogros2/kubernetes/gcp_kubernetes.py +++ b/fogros2/fogros2/kubernetes/gcp_kubernetes.py @@ -37,44 +37,47 @@ import subprocess import time import uuid +import tempfile -from .cloud_instance import CloudInstance +from ..util import extract_bash_column +from ..cloud_instance import CloudInstance -class GCPCloudInstance(CloudInstance): + +class GCPKubeInstance(CloudInstance): """Kubernetes Implementation of CloudInstance.""" def __init__( self, - project_id, container_image='ubuntu', zone="us-central1-a", - mCpu=500, - mB=200, + mcpu=500, + mb=200, **kwargs, ): super().__init__(**kwargs) - self.cloud_service_provider = "Kubernetes" + self.cloud_service_provider = "GKE" id_ = str(uuid.uuid4())[0:8] self._name = f'fog-{id_}-{self._name}' self.zone = zone - self.type = machine_type + self.type = f'{mcpu}mx{mb}Mb' self.container_image = container_image + self._mcpu = mcpu + self._mmb = mb + self._working_dir = os.path.join(self._working_dir_base, self._name) os.makedirs(self._working_dir, exist_ok=True) - self._project_id = project_id - # after config self._ssh_key = None self.create() def create(self): - self.logger.info(f"Creating new GCP Compute Engine instance with name {self._name}") + self.logger.info(f"Creating new Kubernetes Pod with name {self._name}") self.create_compute_engine_instance() self.info(flush_to_disk=True) self.connect() @@ -89,38 +92,113 @@ def info(self, flush_to_disk=True): info_dict = super().info(flush_to_disk) info_dict["compute_region"] = self.zone info_dict["compute_instance_type"] = self.type - info_dict["disk_size"] = self.compute_instance_disk_size info_dict["compute_instance_id"] = self._name if flush_to_disk: with open(os.path.join(self._working_dir, "info"), "w+") as f: json.dump(info_dict, f) return info_dict + def create_service_pair(self, kube_wd: str, pub_key_path: str): + ssh_config: dict = json.loads(open(f'{kube_wd}/ssh.json').read()) + vpn_config: dict = json.loads(open(f'{kube_wd}/vpn.json').read()) + pod_config: dict = json.loads(open(f'{kube_wd}/pod.json').read()) + + # Configure the pod + pod_config['spec']['containers'][0].image = self.container_image + pod_config['spec']['containers'][0].name = self._name + pod_config['spec']['containers'][0]['resources']['requests']['memory'] = str(self._mmb) + 'Mi' + pod_config['spec']['containers'][0]['resources']['requests']['cpu'] = str(self._mcpu) + "m" + + pod_config['spec']['containers'][0]['resources']['limits']['memory'] = str(self._mmb) + "Mi" + pod_config['spec']['containers'][0]['resources']['limits']['cpu'] = str(self._mcpu) + "m" + + pod_config['spec']['containers'][0].env.append({ + 'name': "SSH_PUBKEY", + 'value': open(pub_key_path).read() + }) + + pod_config['metadata'] = { + 'labels': { + 'app': self._name + } + } + + # Configure SSH + ssh_config['metadata']['name'] = f'{self._name}-ssh' + ssh_config['selector']['app'] = self._name + + # Configure VPN + vpn_config['metadata']['name'] = f'{self._name}-vpn' + vpn_config['selector']['app'] = self._name + + ssh_tempfile = tempfile.NamedTemporaryFile() + ssh_tempfile.write(json.dumps(ssh_config)) + vpn_tempfile = tempfile.NamedTemporaryFile() + vpn_tempfile.write(json.dumps(vpn_config)) + pod_tempfile = tempfile.NamedTemporaryFile() + pod_tempfile.write(json.dumps(pod_config)) + + print("Creating SSH...") + os.system(f"kubectl apply -f {ssh_tempfile.name}") + print("Creating VPN...") + os.system(f"kubectl apply -f {vpn_tempfile.name}") + print("Creating Pod") + os.system(f"kubectl apply -f {pod_tempfile.name}") + + ssh_tempfile.close() + vpn_tempfile.close() + pod_tempfile.close() + + # Wait until all services are live + while True: + if 'ContainerCreating' in \ + subprocess.check_output(f'kubectl get pod {self._name}', shell=True).decode() or \ + 'pending' in \ + subprocess.check_output(f'kubectl get service {vpn_config["metadata"]["name"]}', shell=True). \ + decode() \ + or 'pending' in \ + subprocess.check_output(f'kubectl get service {ssh_config["metadata"]["name"]}', shell=True). \ + decode(): + print("Some services still creating...") + time.sleep(1) + else: + break + + # Setup ssh + command = f'useradd "{self._username}" -m -s /bin/bash && ' \ + f'mkdir "/home/{self._username}/.ssh" && ' \ + f'echo "$SSH_PUBKEY" >> "/home/{self._username}/.ssh/authorized_keys" && ' \ + f'chmod -R u=rwX "/home/{self._username}/.ssh" && ' \ + f'chown -R "{self._username}:{self._username}" "/home/{self._username}/.ssh" && ' \ + f'echo "{self._username} ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/shade && ' \ + f'service ssh restart && ' \ + 'sleep infinity' + + print("Enabling SSH") + subprocess.check_output(f'kubectl exec strategy -- /bin/bash -c "{command}"', shell=True) + + print("Extracting IPs") + ssh_data = subprocess.check_output(f'kubectl get service {ssh_config["metadata"]["name"]}', shell=True).decode() + vpn_data = subprocess.check_output(f'kubectl get service {vpn_config["metadata"]["name"]}', shell=True).decode() + + return extract_bash_column(ssh_data, 'EXTERNAL-IP'), extract_bash_column(vpn_data, 'EXTERNAL-IP') + def create_compute_engine_instance(self): - user = subprocess.check_output('whoami').decode().strip() + # Generate SSH keys + user = subprocess.check_output('whoami', shell=True).decode().strip() + kube_wd = f'/home/{user}/fog_ws/src/FogROS2/fogros2/fogros2/kubernetes' - # Username - self._username = (open(f'/home/{user}/.ssh/google_compute_engine.pub'). - read()).split(' ')[-1].strip().split('@')[0] + self._ssh_key = f'/home/{user}/.ssh/{self._name}' + os.system(f"ssh-keygen -f {self._ssh_key} -q -N ''") - self._ssh_key_path = f'/home/{user}/.ssh/google_compute_engine' - self._is_created = True + ssh_ip, vpn_ip = self.create_service_pair(kube_wd, f'{self._ssh_key}.pub') + + self._ip = ssh_ip + self._vpn_ip = vpn_ip self.logger.info( f"Created {self.type} instance named {self._name} " - f"with id {self._name} and public IP address {self._ip}" + f"with id {self._name} and public IP address {self._ip} with VPN ip {self._vpn_ip}" ) - command = 'useradd "$USERNAME" -m -s /bin/bash && ' \ - 'mkdir "/home/$USERNAME/.ssh" && ' \ - 'echo "$SSH_PUBKEY" >> "/home/$USERNAME/.ssh/authorized_keys" && ' \ - 'chmod -R u=rwX "/home/$USERNAME/.ssh" && ' \ - 'chown -R "$USERNAME:$USERNAME" "/home/$USERNAME/.ssh" && ' \ - 'echo "$USERNAME ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/shade && ' \ - 'service ssh restart && ' \ - 'wg-quick the vpn configuration && ' \ - 'echo "$VPN_CONFIG" | base64 --decode > /home/app/vpn.conf && ' \ - 'cp /home/app/vpn.conf /etc/wireguard/wg0.conf && ' \ - 'chmod 600 /etc/wireguard/wg0.conf && ' \ - 'sudo wg-quick up wg0 && ' \ - 'sleep infinity' + self._is_created = True diff --git a/fogros2/fogros2/kubernetes/pod.json b/fogros2/fogros2/kubernetes/pod.json index 347e318..08bced7 100644 --- a/fogros2/fogros2/kubernetes/pod.json +++ b/fogros2/fogros2/kubernetes/pod.json @@ -2,7 +2,7 @@ "apiVersion": "v1", "kind": "Pod", "metadata": { - "name": "strategy" + "name": "fogros" }, "spec": { "restartPolicy": "Never", diff --git a/fogros2/fogros2/kubernetes/ssh.json b/fogros2/fogros2/kubernetes/ssh.json new file mode 100644 index 0000000..a84bc65 --- /dev/null +++ b/fogros2/fogros2/kubernetes/ssh.json @@ -0,0 +1,22 @@ +{ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "ssh-balancer", + "namespace": "default" + }, + "spec": { + "type": "LoadBalancer", + "ports": [ + { + "port": 22, + "targetPort": 22, + "name": "ssh", + "protocol": "TCP" + } + ], + "selector": { + "app": "ssh-pod" + } + } +} \ No newline at end of file diff --git a/fogros2/fogros2/kubernetes/ssh.yaml b/fogros2/fogros2/kubernetes/ssh.yaml deleted file mode 100644 index 25c1368..0000000 --- a/fogros2/fogros2/kubernetes/ssh.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ssh-balancer - namespace: ros -spec: - type: LoadBalancer - ports: - - port: 22 - targetPort: 22 - name: ssh - protocol: TCP - selector: - app: ssh-pod diff --git a/fogros2/fogros2/kubernetes/vpn.json b/fogros2/fogros2/kubernetes/vpn.json new file mode 100644 index 0000000..b374a9b --- /dev/null +++ b/fogros2/fogros2/kubernetes/vpn.json @@ -0,0 +1,22 @@ +{ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "vpn-balancer", + "namespace": "default" + }, + "spec": { + "type": "LoadBalancer", + "ports": [ + { + "port": 51820, + "targetPort": 51820, + "name": "vpn", + "protocol": "UDP" + } + ], + "selector": { + "app": "ssh-pod" + } + } +} \ No newline at end of file diff --git a/fogros2/fogros2/kubernetes/vpn.yaml b/fogros2/fogros2/kubernetes/vpn.yaml deleted file mode 100644 index 5147cfc..0000000 --- a/fogros2/fogros2/kubernetes/vpn.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: vpn-balancer - namespace: ros -spec: - type: LoadBalancer - ports: - - port: 51820 - targetPort: 51820 - name: vpn - protocol: UDP - selector: - app: ssh-pod diff --git a/fogros2/fogros2/util.py b/fogros2/fogros2/util.py index f7125c1..5b7ca96 100644 --- a/fogros2/fogros2/util.py +++ b/fogros2/fogros2/util.py @@ -79,3 +79,27 @@ def make_zip_file(dir_name, target_path): format="zip", base_name=target_path, ) + + +def extract_bash_column(subprocess_output: str, column_name: str, row_number: int = 0): + """ + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + ssh-balancer LoadBalancer 10.0.0.15 22:32695/TCP 19s + + This util finds the value of any given column value - ex: CLUSTER-IP -> 10.0.015 + :param subprocess_output: Direct output of subprocess.check_output().decode() + :param column_name: The column name to search for ex: CLUSTER-IP + :param row_number: Defaults to the first data row, row_number = 1 is second data row + :return: String of output value + """ + lines = subprocess_output.split('\n') + if column_name not in lines[0]: + raise LookupError(f"Could not find column {column_name} in {lines[0].strip()}") + column_index = lines[0].index(column_name) + + output_str = '' + while column_index != len(lines[row_number+1]) and lines[row_number+1][column_index] != ' ': + output_str += lines[row_number+1][column_index] + column_index += 1 + + return output_str diff --git a/fogros2/fogros2/vpn.py b/fogros2/fogros2/vpn.py index 212f309..47d2ca7 100755 --- a/fogros2/fogros2/vpn.py +++ b/fogros2/fogros2/vpn.py @@ -92,7 +92,10 @@ def generate_wg_config_files(self, machines): robot_config.add_attr(None, "Address", "10.0.0.1/24") for machine in machines: name = machine.name - ip = machine.ip + if hasattr(machine, 'vpn_ip') and machine.vpn_ip is not None: + ip = machine.vpn_ip + else: + ip = machine.ip cloud_pub_key = self.cloud_name_to_pub_key_path[name] robot_config.add_peer(cloud_pub_key, f"# AWS{name}") robot_config.add_attr(cloud_pub_key, "AllowedIPs", "10.0.0.2/32") diff --git a/fogros2_examples/launch/talker.gcp.launch.py b/fogros2_examples/launch/talker.gcp.launch.py index 31156e8..b45faad 100644 --- a/fogros2_examples/launch/talker.gcp.launch.py +++ b/fogros2_examples/launch/talker.gcp.launch.py @@ -37,7 +37,7 @@ def generate_launch_description(): - """Talker example that launches the listener on AWS.""" + """Talker example that launches the listener on Google Compute Engine.""" ld = fogros2.FogROSLaunchDescription() machine1 = fogros2.GCPCloudInstance( project_id='shade-prod' diff --git a/fogros2_examples/launch/talker.kube.launch.py b/fogros2_examples/launch/talker.kube.launch.py new file mode 100644 index 0000000..f9704f3 --- /dev/null +++ b/fogros2_examples/launch/talker.kube.launch.py @@ -0,0 +1,56 @@ +# Copyright 2022 The Regents of the University of California (Regents) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright ©2022. The Regents of the University of California (Regents). +# All Rights Reserved. Permission to use, copy, modify, and distribute this +# software and its documentation for educational, research, and not-for-profit +# purposes, without fee and without a signed licensing agreement, is hereby +# granted, provided that the above copyright notice, this paragraph and the +# following two paragraphs appear in all copies, modifications, and +# distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150 +# Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201, +# otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial +# licensing opportunities. IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY +# FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, +# INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS +# DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +# DAMAGE. REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, +# PROVIDED HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +# MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +from launch_ros.actions import Node + +import fogros2 + + +def generate_launch_description(): + """Talker example that launches the listener on GCP Kube.""" + ld = fogros2.FogROSLaunchDescription() + machine1 = fogros2.GCPKubeInstance() + + listener_node = Node( + package="fogros2_examples", executable="listener", output="screen" + ) + + talker_node = fogros2.CloudNode( + package="fogros2_examples", + executable="talker", + output="screen", + machine=machine1, + ) + ld.add_action(talker_node) + ld.add_action(listener_node) + return ld From fa8b1ea6dea2aff526abe090d0603cd8e7c38946 Mon Sep 17 00:00:00 2001 From: Emerson Dove <52636744+EmersonDove@users.noreply.github.com> Date: Wed, 20 Jul 2022 10:39:11 -0400 Subject: [PATCH 5/6] Fix gcloud formatting --- fogros2/fogros2/gcp_cloud_instance.py | 52 ++++++++++++++------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/fogros2/fogros2/gcp_cloud_instance.py b/fogros2/fogros2/gcp_cloud_instance.py index deb300d..1178278 100644 --- a/fogros2/fogros2/gcp_cloud_instance.py +++ b/fogros2/fogros2/gcp_cloud_instance.py @@ -35,11 +35,12 @@ import os import subprocess -import time import uuid from .cloud_instance import CloudInstance +from .util import extract_bash_column + class GCPCloudInstance(CloudInstance): """GCP Implementation of CloudInstance.""" @@ -100,29 +101,30 @@ def info(self, flush_to_disk=True): def create_compute_engine_instance(self): os.system(f'gcloud config set project {self._project_id}') - ip = subprocess.check_output(f'gcloud compute instances create {self._name} ' - f'--project={self._project_id} --zone={self.zone} --machine-type={self.type} ' - '--network-interface=network-tier=PREMIUM,subnet=default ' - '--maintenance-policy=MIGRATE --provisioning-model=STANDARD ' - '--scopes=https://www.googleapis.com/auth/devstorage.read_only,' - 'https://www.googleapis.com/auth/logging.write,' - 'https://www.googleapis.com/auth/monitoring.write,' - 'https://www.googleapis.com/auth/servicecontrol,' - 'https://www.googleapis.com/auth/service.management.readonly,' - 'https://www.googleapis.com/auth/trace.append ' - '--create-disk=auto-delete=yes,' - 'boot=yes,' - f'device-name={self._name},' - f'image={self.gcp_ami_image},' - 'mode=rw,' - f'size={self.compute_instance_disk_size},' - f'type=projects/{self._project_id}/zones/{self.zone}/diskTypes/pd-balanced ' - '--no-shielded-secure-boot ' - '--shielded-vtpm ' - '--shielded-integrity-monitoring ' - '--reservation-affinity=any | ' - "awk '{print $5}' | " - "sed -n 2p", shell=True).decode().strip() + result = subprocess.check_output(f'gcloud compute instances create {self._name} ' + f'--project={self._project_id} --zone={self.zone} --machine-type={self.type} ' + '--network-interface=network-tier=PREMIUM,subnet=default ' + '--maintenance-policy=MIGRATE --provisioning-model=STANDARD ' + '--scopes=https://www.googleapis.com/auth/devstorage.read_only,' + 'https://www.googleapis.com/auth/logging.write,' + 'https://www.googleapis.com/auth/monitoring.write,' + 'https://www.googleapis.com/auth/servicecontrol,' + 'https://www.googleapis.com/auth/service.management.readonly,' + 'https://www.googleapis.com/auth/trace.append ' + '--create-disk=auto-delete=yes,' + 'boot=yes,' + f'device-name={self._name},' + f'image={self.gcp_ami_image},' + 'mode=rw,' + f'size={self.compute_instance_disk_size},' + f'type=projects/{self._project_id}/zones/{self.zone}/diskTypes/pd-balanced ' + '--no-shielded-secure-boot ' + '--shielded-vtpm ' + '--shielded-integrity-monitoring ' + '--reservation-affinity=any', shell=True).decode() + + # Grab external IP + ip = extract_bash_column(result, 'EXTERNAL_IP') # Verifies the response was an ip if len(ip.split('.')) != 4: @@ -133,7 +135,7 @@ def create_compute_engine_instance(self): # Generate SSH keys os.system(f"printf '\n\n' | gcloud compute ssh {self._name} --zone {self.zone}") - user = subprocess.check_output('whoami').decode().strip() + user = subprocess.check_output('whoami', shell=True).decode().strip() # Username self._username = (open(f'/home/{user}/.ssh/google_compute_engine.pub'). From 8121eee5bbe78f4c2025c5d58342bbcb3c3ea113 Mon Sep 17 00:00:00 2001 From: Emerson Dove <52636744+EmersonDove@users.noreply.github.com> Date: Wed, 20 Jul 2022 17:19:42 +0000 Subject: [PATCH 6/6] Correct kube integration issues --- fogros2/fogros2/cloud_instance.py | 3 +- fogros2/fogros2/kubernetes/gcp_kubernetes.py | 50 ++++++++------------ fogros2/fogros2/kubernetes/pod.json | 4 +- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/fogros2/fogros2/cloud_instance.py b/fogros2/fogros2/cloud_instance.py index b37bd27..eb40ec4 100644 --- a/fogros2/fogros2/cloud_instance.py +++ b/fogros2/fogros2/cloud_instance.py @@ -132,7 +132,7 @@ def apt_install(self, args): ) def pip_install(self, args): - self.scp.execute_cmd(f"sudo pip3 install {args}") + self.scp.execute_cmd(f"python3 -m pip install {args}") def install_cloud_dependencies(self): self.apt_install("wireguard unzip docker.io python3-pip") @@ -168,6 +168,7 @@ def install_ros(self): # install ros2 packages self.apt_install(f"ros-{self.ros_distro}-desktop") + # Installing all deps because cloud launch seems to rely on them self.apt_install('python3-colcon-common-extensions') # source environment diff --git a/fogros2/fogros2/kubernetes/gcp_kubernetes.py b/fogros2/fogros2/kubernetes/gcp_kubernetes.py index bdbf508..497bde7 100644 --- a/fogros2/fogros2/kubernetes/gcp_kubernetes.py +++ b/fogros2/fogros2/kubernetes/gcp_kubernetes.py @@ -51,8 +51,8 @@ def __init__( self, container_image='ubuntu', zone="us-central1-a", - mcpu=500, - mb=200, + mcpu=0, + mb=0, **kwargs, ): super().__init__(**kwargs) @@ -104,20 +104,21 @@ def create_service_pair(self, kube_wd: str, pub_key_path: str): pod_config: dict = json.loads(open(f'{kube_wd}/pod.json').read()) # Configure the pod - pod_config['spec']['containers'][0].image = self.container_image - pod_config['spec']['containers'][0].name = self._name + pod_config['spec']['containers'][0]["image"] = self.container_image + pod_config['spec']['containers'][0]["name"] = self._name pod_config['spec']['containers'][0]['resources']['requests']['memory'] = str(self._mmb) + 'Mi' pod_config['spec']['containers'][0]['resources']['requests']['cpu'] = str(self._mcpu) + "m" pod_config['spec']['containers'][0]['resources']['limits']['memory'] = str(self._mmb) + "Mi" pod_config['spec']['containers'][0]['resources']['limits']['cpu'] = str(self._mcpu) + "m" - pod_config['spec']['containers'][0].env.append({ + pod_config['spec']['containers'][0]["env"].append({ 'name': "SSH_PUBKEY", - 'value': open(pub_key_path).read() + 'value': open(pub_key_path).read().strip() }) pod_config['metadata'] = { + 'name': self._name, 'labels': { 'app': self._name } @@ -125,18 +126,18 @@ def create_service_pair(self, kube_wd: str, pub_key_path: str): # Configure SSH ssh_config['metadata']['name'] = f'{self._name}-ssh' - ssh_config['selector']['app'] = self._name + ssh_config['spec']['selector']['app'] = self._name # Configure VPN vpn_config['metadata']['name'] = f'{self._name}-vpn' - vpn_config['selector']['app'] = self._name + vpn_config['spec']['selector']['app'] = self._name ssh_tempfile = tempfile.NamedTemporaryFile() - ssh_tempfile.write(json.dumps(ssh_config)) + open(ssh_tempfile.name, 'w').write(json.dumps(ssh_config)) vpn_tempfile = tempfile.NamedTemporaryFile() - vpn_tempfile.write(json.dumps(vpn_config)) + open(vpn_tempfile.name, 'w').write(json.dumps(vpn_config)) pod_tempfile = tempfile.NamedTemporaryFile() - pod_tempfile.write(json.dumps(pod_config)) + open(pod_tempfile.name, 'w').write(json.dumps(pod_config)) print("Creating SSH...") os.system(f"kubectl apply -f {ssh_tempfile.name}") @@ -160,23 +161,10 @@ def create_service_pair(self, kube_wd: str, pub_key_path: str): subprocess.check_output(f'kubectl get service {ssh_config["metadata"]["name"]}', shell=True). \ decode(): print("Some services still creating...") - time.sleep(1) + time.sleep(2) else: break - # Setup ssh - command = f'useradd "{self._username}" -m -s /bin/bash && ' \ - f'mkdir "/home/{self._username}/.ssh" && ' \ - f'echo "$SSH_PUBKEY" >> "/home/{self._username}/.ssh/authorized_keys" && ' \ - f'chmod -R u=rwX "/home/{self._username}/.ssh" && ' \ - f'chown -R "{self._username}:{self._username}" "/home/{self._username}/.ssh" && ' \ - f'echo "{self._username} ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/shade && ' \ - f'service ssh restart && ' \ - 'sleep infinity' - - print("Enabling SSH") - subprocess.check_output(f'kubectl exec strategy -- /bin/bash -c "{command}"', shell=True) - print("Extracting IPs") ssh_data = subprocess.check_output(f'kubectl get service {ssh_config["metadata"]["name"]}', shell=True).decode() vpn_data = subprocess.check_output(f'kubectl get service {vpn_config["metadata"]["name"]}', shell=True).decode() @@ -188,17 +176,19 @@ def create_compute_engine_instance(self): user = subprocess.check_output('whoami', shell=True).decode().strip() kube_wd = f'/home/{user}/fog_ws/src/FogROS2/fogros2/fogros2/kubernetes' - self._ssh_key = f'/home/{user}/.ssh/{self._name}' - os.system(f"ssh-keygen -f {self._ssh_key} -q -N ''") + self._ssh_key_path = f'/home/{user}/.ssh/{self._name}' + os.system(f"ssh-keygen -f {self._ssh_key_path} -q -N ''") - ssh_ip, vpn_ip = self.create_service_pair(kube_wd, f'{self._ssh_key}.pub') + ssh_ip, vpn_ip = self.create_service_pair(kube_wd, f'{self._ssh_key_path}.pub') self._ip = ssh_ip self._vpn_ip = vpn_ip + self._username = 'ubuntu' + + self._is_created = True + self.logger.info( f"Created {self.type} instance named {self._name} " f"with id {self._name} and public IP address {self._ip} with VPN ip {self._vpn_ip}" ) - - self._is_created = True diff --git a/fogros2/fogros2/kubernetes/pod.json b/fogros2/fogros2/kubernetes/pod.json index 08bced7..fb69423 100644 --- a/fogros2/fogros2/kubernetes/pod.json +++ b/fogros2/fogros2/kubernetes/pod.json @@ -26,7 +26,9 @@ "cpu": "<125m means 125 millicpu>" } }, - "env": [] + "env": [], + "command": ["/bin/bash"], + "args": ["-c", "apt update && apt install -y openssh-server sudo curl && useradd 'ubuntu' -m -s /bin/bash && mkdir '/home/ubuntu/.ssh' && echo $SSH_PUBKEY >> '/home/ubuntu/.ssh/authorized_keys' && chmod -R u=rwX '/home/ubuntu/.ssh' && chown -R 'ubuntu:ubuntu' '/home/ubuntu/.ssh' && echo 'ubuntu ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers && service ssh restart && sleep infinity"] } ] }