Merge branch 'main' into compathelper/new_version/2024-11-27-03-22-11-231-00492685430

Author: decarvalhojunior-fh

Project.toml

@@ -22,14 +22,18 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
 [compat]
+AWS = "1"
 Base64 = "1.10.4"
 Distributed = "1.10.4"
 FilePathsBase = "0.9.21"
+JSON = "0.21.4"
 MPIClusterManagers = "0.2.4"
 PlatformAware = "0.6.1"
 Random = "1.10.4"
 Reexport = "1"
 Serialization = "1.10.4"
 Sockets = "1.10.4"
 TOML = "1.0.3"
-Test = "1.11.0"
+Test = "1.11.0" 
+YAML = "0.4"
+julia = "1"

docs/src/index.md

@@ -25,10 +25,15 @@ _CloudClusters.jl_ assumes that the user has configured their credentials for th
 Creating clusters with _CloudClusters.jl_ requires specifying some configuration parameters. By default, they are specified in a file named _CCconfig.toml_ that is searched in the following locations, in this order:
 * a path pointed by the CLOUD_CLUSTERS_CONFIG environment variable, if it exists;
 * the current path.
-  
+
+Section [Configuration parameters](https://github.com/PlatformAwareProgramming/CloudClusters.jl#configuration-parameters) describes configuration parameters and how they can be overridden in programs.  
+
 Default configuration parameters can be overridden in programs. 
 
-Section [Configuration parameters](https://github.com/PlatformAwareProgramming/CloudClusters.jl#configuration-parameters) describes configuration parameters and how they can be overridden in programs.
+A [_CCconfig.toml_](https://raw.githubusercontent.com/PlatformAwareProgramming/CloudClusters.jl/refs/heads/main/CCconfig.toml) file is provided in the repository's top-level directory. It is configured to create clusters using prebuilt virtual machine images for each supported cloud provider. These images are based on the latest version of Ubuntu and include a Julia installation of a recent stable version with all the packages needed to instantiate the clusters added and precompiled. Users can create customized images, possibly derived from the provided image, using their preferred version of Julia and adding the packages they need. 
+
+> [!NOTE]
+> The version of Julia on the host computer using _CloudClusters.jl_ must be the same version as the image used to deploy the clusters.
 
 ### The _PlatformAware.jl_ package
 
@@ -192,7 +197,7 @@ julia> @nodes my_second_cluster
 12
 13
 
-julia> @nodes my_third_cluster
+julia> @ nodes my_third_cluster
 8-element Vector{Int64}
 14
 15

src/CloudClusters.jl

@@ -7,6 +7,7 @@ using Reexport
 using Base.Threads
 using Dates
 using TOML
+using AWS
 
 include("config/configs.jl") 
 include("features/features.jl")

src/cluster_providers/ec2/ec2_backend.jl

@@ -3,32 +3,6 @@
 For this module to work, you`ll need AWS credentials in the ${HOME}/.aws directory. 
 =# 
 
-#=
-1. Create placement group.
-2. Create EFS Filesystem.
-3. Create EC2 instances and attach them to the EFS.
-=# 
-
-using AWS
-println("Adaptando o código do Módulo AWS.jl...")
-using FilePathsBase
-aws_package_dir = ENV["HOME"] * "/.julia/packages/AWS"
-all_entries = readdir(aws_package_dir)
-subdirs = filter(entry -> isdir(joinpath(aws_package_dir, entry)), all_entries)
-
-for subdir in subdirs
-    ec2_file = joinpath(aws_package_dir, subdir, "src", "services", "ec2.jl")
-    chmod(ec2_file, 0o644)
-    content = read(ec2_file, String)
-    new_content = replace(content, "Dict{String,Any}(\"groupName\" => groupName);" => "Dict{String,Any}(\"GroupName\" => groupName);")
-    new_content = replace(new_content, "\"MaxCount\" => MaxCount, \"MinCount\" => MinCount, \"clientToken\" => string(uuid4())" => 
-                                       "\"MaxCount\" => MaxCount, \"MinCount\" => MinCount, \"ClientToken\" => string(uuid4())")
-    new_content = replace(new_content, "\"clientToken\" => string(uuid4())" =>  "\"ClientToken\" => string(uuid4())")
-    open(ec2_file, "w") do io
-        write(io, new_content)
-    end
-end
-
 using Random
 using AWS: @service
 using Serialization
@@ -113,7 +87,6 @@ function ec2_create_cluster(cluster::Cluster)
 end
 
 
-
 function ec2_get_ips_instance(instance_id::String)
     public_ip = Ec2.describe_instances(Dict("InstanceId" => instance_id))["reservationSet"]["item"]["instancesSet"]["item"]["ipAddress"]
     private_ip = Ec2.describe_instances(Dict("InstanceId" => instance_id))["reservationSet"]["item"]["instancesSet"]["item"]["privateIpAddress"]
@@ -150,7 +123,7 @@ end
 
 function ec2_delete_placement_group(name)
     params = Dict("GroupName" => name)
-    Ec2.delete_placement_group(name)
+    delete_placement_group(name)
 end
 
 #=
@@ -195,8 +168,6 @@ end
 Criação de Instâncias
 =# 
 
-
-
 # Funções auxiliares.
 function ec2_set_up_ssh_connection(cluster_name)  
 
@@ -382,14 +353,14 @@ chown -R ubuntu:ubuntu /home/ubuntu/shared
     # Criando as instâncias
     params_manager, params_workers = ec2_create_params(cluster, user_data_base64)
     # Criar o headnode
-    instance_headnode = Ec2.run_instances(1, 1, params_manager)
+    instance_headnode = run_instances(1, 1, params_manager)
     cluster_nodes[:manager] = instance_headnode["instancesSet"]["item"]["instanceId"]
 
     # Criar os worker nodes.
     params_workers["InstanceType"] = cluster.instance_type_worker
     params_workers["TagSpecification"]["Tag"][2]["Value"] = "worker"
     count = cluster.count
-    instances_workers = Ec2.run_instances(count, count, params_workers)
+    instances_workers = run_instances(count, count, params_workers)
     workers = count
     for i in 1:count
         instance = ""
@@ -435,7 +406,7 @@ chown -R ubuntu:ubuntu /home/ubuntu/shared
 
     # Criar os Peers.
     count = cluster.count
-    instances_peers = Ec2.run_instances(count, count, params)
+    instances_peers = run_instances(count, count, params)
     for i in 1:count
         instance = ""
         if count > 1
@@ -640,4 +611,68 @@ function ec2_get_ips(cluster::Cluster)
          ips[node] = ec2_get_ips_instance(id) 
     end
     ips
-end
+end
+
+
+
+function delete_placement_group(
+    groupName; aws_config::AbstractAWSConfig=global_aws_config()
+)
+    return Ec2.ec2(
+        "DeletePlacementGroup",
+        Dict{String,Any}("GroupName" => groupName);
+        aws_config=aws_config,
+        feature_set=Ec2.SERVICE_FEATURE_SET,
+    )
+end
+function delete_placement_group(
+    groupName,
+    params::AbstractDict{String};
+    aws_config::AbstractAWSConfig=global_aws_config(),
+)
+    return Ec2.ec2(
+        "DeletePlacementGroup",
+        Dict{String,Any}(
+            mergewith(_merge, Dict{String,Any}("GroupName" => groupName), params)
+        );
+        aws_config=aws_config,
+        feature_set= Ec2.SERVICE_FEATURE_SET,
+    )
+end
+
+function run_instances(
+    MaxCount, MinCount; aws_config::AbstractAWSConfig=global_aws_config()
+)
+    return Ec2.ec2(
+        "RunInstances",
+        Dict{String,Any}(
+            "MaxCount" => MaxCount, "MinCount" => MinCount, "ClientToken" => string(Ec2.uuid4())
+        );
+        aws_config=aws_config,
+        feature_set=Ec2.SERVICE_FEATURE_SET,
+    )
+end
+function run_instances(
+    MaxCount,
+    MinCount,
+    params::AbstractDict{String};
+    aws_config::AbstractAWSConfig=global_aws_config(),
+)
+    return Ec2.ec2(
+        "RunInstances",
+        Dict{String,Any}(
+            mergewith(
+                _merge,
+                Dict{String,Any}(
+                    "MaxCount" => MaxCount,
+                    "MinCount" => MinCount,
+                    "ClientToken" => string(Ec2.uuid4()),
+                ),
+                params,
+            ),
+        );
+        aws_config=aws_config,
+        feature_set=Ec2.SERVICE_FEATURE_SET,
+    )
+end
+