GCP support, @status, @config, minor bug corrections

Author: decarvalhojunior-fh

CCconfig.toml

@@ -18,18 +18,20 @@ mpiflags = ""
 
 [ec2]
 
-imageid = "ami-0b869698add04fbdc"  # found at us-east-1 (North Virginia). To use in other regions, copy it.
+imageid = "ami-0bec2868f8f28086f"  # found at us-east-1 (North Virginia). To use in other regions, copy it.
+#security_group_id = "sg-09e2e7c3eebd45160"
 
 # placement_group = "pg-XXXXXXXXXXXX" or "automatic"
 # security_group_id = "sg-XXXXXXXXXXXX" or "automatic"
 # subnet_id = "subnet-XXXXXXXXXXXX"
 
 [gcp]
 
-imageid = "hpc-shelf-311900/global/images/cloudclusters-basic-v3"
+imageid = "hpc-shelf-311900/global/images/cloudclusters-basic-v5"
 zone = "us-central1-a"
 project = "hpc-shelf-311900"
 user = "heron"
 exename = "/home/heron/.juliaup/bin/julia"
 directory = "/home/heron"
 mpiflags = "--map-by node --hostfile /home/heron/hostfile"
+# network_interface = "default"

Project.toml

@@ -1,7 +1,7 @@
 name = "CloudClusters"
 uuid = "4ca6f12b-c8f1-4945-b50f-6bb73234c039"
 authors = ["Francisco Heron de Carvalho Junior <heron@dc.ufc.br> e João Marcelo Uchôa de Alencar <joao.marcelo@ufc.br>"]
-version = "0.1.2"
+version = "0.2.0"
 
 [deps]
 AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"

README.md

@@ -46,10 +46,10 @@ export cluster_create, @cluster,
        cluster_restart, @restart,
        cluster_features, @features,
        cluster_nodes, @nodes,
-       cluster_defaultconfig,
-       cluster_providers,
-       cluster_features
-
+       cluster_status, @status,
+       cluster_config, @config,
+       cluster_providers, @providers
+       
 # Cluster types
 export ManagerWorkers, PeerWorkers, PeerWorkersMPI, Localhost
 

docs/src/index.md

@@ -66,7 +66,7 @@ function cluster_list(;from = DateTime(0), cluster_type = :AnyCluster)
     path_contents = readdir(configpath; join = true)
 
     for cluster_file in path_contents
-        if occursin(r"\s*.cluster", cluster_file)
+        if file_extension(cluster_file) == "cluster"
             cluster_data = load_cluster(cluster_file; from=from, cluster_type=cluster_type)
             !isempty(cluster_data) && push!(result, cluster_data)
          end

src/CloudClusters.jl

@@ -135,3 +135,24 @@ cluster_isrunning(_::Type{AmazonEC2}, cluster_handle) = ec2_cluster_info[cluster
 cluster_isstopped(_::Type{AmazonEC2}, cluster_handle) = ec2_cluster_info[cluster_handle] |> ec2_cluster_isstopped
 
 
+function cluster_status(_::Type{AmazonEC2}, cluster_handle)
+  cluster = ec2_cluster_info[cluster_handle] 
+  cluster_nodes = cluster.cluster_nodes
+  error = false
+  cluster_status = nothing
+  for (nodeid,instanceid) in cluster_nodes
+      node_status = ec2_get_instance_status(instanceid)
+      @info "$nodeid ($instanceid) is $node_status"
+      error = !isnothing(cluster_status) && cluster_status != node_status 
+      cluster_status = node_status
+  end
+  if error
+      @error "The EC2 cluster is in a inconsistent status (all nodes must be in the same status)"
+  else
+      @info "The cluster $cluster_handle at EC2 is in $cluster_status status"
+  end
+end
+
+function cluster_delete(_::Type{AmazonEC2}, cluster_handle)
+  ec2_delete_cluster(cluster_handle)
+end

src/cluster.jl

@@ -108,7 +108,6 @@ function cluster_load(_::Type{AmazonEC2}, _::Type{<:ManagerWorkers}, cluster_han
         ec2_cluster_info[cluster_handle] = cluster
         return cluster.features
     else
-        ec2_delete_cluster(cluster_handle)
         return nothing
     end
 end
@@ -155,7 +154,6 @@ function cluster_load(_::Type{AmazonEC2}, _::Type{<:PeerWorkers}, cluster_handle
         ec2_cluster_info[cluster_handle] = cluster
         return cluster.features
     else
-        ec2_delete_cluster(cluster_handle)
         return nothing
     end
 end

src/cluster_providers/ec2/ec2_deploy.jl

@@ -34,6 +34,7 @@ mutable struct GCPManagerWorkers <: ManagerWorkers #Cluster
     user_worker::String
     zone::String
     project::String
+    network_interface::String
     cluster_nodes::Union{Dict{Symbol, String}, Nothing}
     features::Dict{Symbol, Any}
 end
@@ -47,6 +48,7 @@ mutable struct GCPPeerWorkers <: PeerWorkers # Cluster
     user::String
     zone::String
     project::String
+    network_interface::String
     cluster_nodes::Union{Dict{Symbol, String}, Nothing}
     features::Dict{Symbol, Any}
 end
@@ -59,6 +61,7 @@ mutable struct GCPPeerWorkersMPI <: PeerWorkersMPI # Cluster
     user::String
     zone::String
     project::String
+    network_interface::String
     cluster_nodes::Union{Dict{Symbol, String}, Nothing}
     features::Dict{Symbol, Any}
 end
@@ -189,7 +192,7 @@ function gcp_create_params(cluster::ManagerWorkers, cluster_nodes, internal_key_
             "name" => "external-nat",
             "type" => "ONE_TO_ONE_NAT"
         )],
-        "network" => "https://www.googleapis.com/compute/v1/projects/$(cluster.project)/global/networks/default"
+        "network" => "https://www.googleapis.com/compute/v1/projects/$(cluster.project)/global/networks/$(cluster.network_interface)"
     )],
     "metadata" => 
         "items" => [Dict(
@@ -224,7 +227,7 @@ function gcp_create_params(cluster::ManagerWorkers, cluster_nodes, internal_key_
                     "name" => "external-nat",
                     "type" => "ONE_TO_ONE_NAT"
                 )],
-                "network" => "https://www.googleapis.com/compute/v1/projects/$(cluster.project)/global/networks/default"
+                "network" => "https://www.googleapis.com/compute/v1/projects/$(cluster.project)/global/networks/$(cluster.network_interface)"
             )],
             "metadata" => 
                 "items" => [Dict(
@@ -270,7 +273,7 @@ function gcp_create_params(cluster::PeerWorkers, cluster_nodes, internal_key_nam
                     "name" => "external-nat",
                     "type" => "ONE_TO_ONE_NAT"
                 )],
-                "network" => "https://www.googleapis.com/compute/v1/projects/$(cluster.project)/global/networks/default"
+                "network" => "https://www.googleapis.com/compute/v1/projects/$(cluster.project)/global/networks/$(cluster.network_interface)"
             )],
             "metadata" => 
                 "items" => [Dict(
@@ -377,7 +380,7 @@ function gcp_create_instances(cluster::ManagerWorkers)
 
     internal_key_name = cluster.name
 
-    try gcp_allow_ssh(cluster.project) catch end
+    try gcp_allow_ssh(cluster) catch end
 
     # Criando as instâncias
     params_manager, params_workers = gcp_create_params(cluster, cluster_nodes, internal_key_name, (user_data_manager, user_data_worker), private_key, public_key)
@@ -412,7 +415,7 @@ function gcp_create_instances(cluster::PeerWorkers)
 
     internal_key_name = cluster.name
 
-    try gcp_allow_ssh(cluster.project) catch end
+    try gcp_allow_ssh(cluster) catch end
 
     # Criando as instâncias
     params = gcp_create_params(new_cluster, cluster_nodes, internal_key_name, user_data, private_key, public_key)
@@ -568,18 +571,20 @@ function gcp_get_instance_dict(cluster::Cluster, name)
 end
 
 
-function gcp_allow_ssh(project)
+function gcp_allow_ssh(cluster)
     firewall_rule = Dict(
-        "allowed" => [
-            Dict("IPProtocol" => "tcp",
-                "ports" => ["22"])],
-        "direction" => "INGRESS",
+#        "allowed" => [
+#            Dict("IPProtocol" => "tcp",
+#                "ports" => ["22"])],
+#        "direction" => "INGRESS",
         "kind" => "compute#firewall",
-        "name" => "allow-ssh",
-        "network" => "projects/$project/global/networks/default",
-        "priority" => 1000,
-        "selfLink" => "projects/$project/global/firewalls/allow-ssh",
-        "sourceRanges" => ["0.0.0.0/0"]
+#        "name" => "allow-ssh",
+        "name" => "hpcshelf-virtualplatform-network-rules",
+"network" => "projects/$(cluster.project)/global/networks/$(cluster.network_interface)",
+#        "priority" => 1000,
+#        "selfLink" => "projects/$(cluster.project)/global/firewalls/allow-ssh",
+        "selfLink" => "projects/$(cluster.project)/global/firewalls/hpcshelf-virtualplatform-network-rules",
+"sourceRanges" => ["0.0.0.0/0"]
     )
 
     GCPAPI.compute(:Firewall, :insert, project; data=firewall_rule)

src/cluster_providers/ec2/ec2_persist.jl

@@ -37,7 +37,8 @@ function deploy_cluster(gcptype::Type{GoogleCloud},
     #image_id_workers = get(cluster_features, :image_id, defaults_dict[GoogleCloud][:image_id]) 
     #image_id_manager = get(cluster_features, :image_id_manager, defaults_dict[GoogleCloud][:image_id_manager])
     zone = get(cluster_features, :zone, defaults_dict[GoogleCloud][:zone]) 
-    project = defaults_dict[GoogleCloud][:project]
+    project = get(cluster_features, :project, defaults_dict[GoogleCloud][:project])
+    network_interface = get(cluster_features, :network_interface, get(defaults_dict[GoogleCloud], :network_interface, "default"))
     instance_type_manager = instance_type[1]
     instance_type_worker = instance_type[2]
 
@@ -51,6 +52,7 @@ function deploy_cluster(gcptype::Type{GoogleCloud},
                             user_worker,
                             zone, 
                             project, 
+                            network_interface,
                             nothing,
                             cluster_features)
 
@@ -76,7 +78,11 @@ function deploy_cluster(gcptype::Type{GoogleCloud},
     imageid = get(cluster_features, :imageid, defaults_dict[GoogleCloud][:imageid]) 
     user = get(cluster_features, :user, defaults_dict[GoogleCloud][:user]) 
     zone = get(cluster_features, :zone, defaults_dict[GoogleCloud][:zone]) 
-    project = defaults_dict[GoogleCloud][:project]
+    project = get(cluster_features, :project, defaults_dict[GoogleCloud][:project])
+    network_interface = get(cluster_features, :network_interface, get(defaults_dict[GoogleCloud],:network_interface, "default"))
+
+
+     # get(cluster_features, :placement_group, get(defaults_dict[AmazonEC2], :placement_group, nothing))
 
     cluster = gcp_build_clusterobj(cluster_type, 
                                    string(cluster_handle), 
@@ -86,6 +92,7 @@ function deploy_cluster(gcptype::Type{GoogleCloud},
                                    user,
                                    zone, 
                                    project, 
+                                   network_interface,
                                    nothing,
                                    cluster_features)
 
@@ -98,11 +105,11 @@ function deploy_cluster(gcptype::Type{GoogleCloud},
     return cluster
 end
 
-gcp_build_clusterobj(_::Type{<:PeerWorkers}, name, image_id, count, instance_type, user, zone, project, cluster_nodes, features) =  
-                                             GCPPeerWorkers(name, image_id, count, instance_type, user, zone, project, cluster_nodes, features)
+gcp_build_clusterobj(_::Type{<:PeerWorkers}, name, image_id, count, instance_type, user, zone, project, network_interface, cluster_nodes, features) =  
+                                             GCPPeerWorkers(name, image_id, count, instance_type, user, zone, project, network_interface, cluster_nodes, features)
 
-gcp_build_clusterobj(_::Type{<:PeerWorkersMPI}, name, image_id, count, instance_type, user, zone, project, cluster_nodes, features) =  
-                                                GCPPeerWorkersMPI(name, image_id, count, instance_type, user, zone, project, cluster_nodes, features)
+gcp_build_clusterobj(_::Type{<:PeerWorkersMPI}, name, image_id, count, instance_type, user, zone, project, network_interface, cluster_nodes, features) =  
+                                                GCPPeerWorkersMPI(name, image_id, count, instance_type, user, zone, project, network_interface, cluster_nodes, features)
 
 function launch_processes(_::Type{GoogleCloud}, cluster_type::Type{<:Cluster}, cluster_handle, ips)
     cluster = gcp_cluster_info[cluster_handle]
@@ -151,4 +158,27 @@ function cluster_isrunning(_::Type{GoogleCloud}, cluster_handle)
         @warn "Erro ao verificar o status do cluster: ", e
         return false
     end
+end
+
+
+function cluster_status(_::Type{GoogleCloud}, cluster_handle)
+    cluster = gcp_cluster_info[cluster_handle] 
+    cluster_nodes = cluster.cluster_nodes
+    error = false
+    cluster_status = nothing
+    for (nodeid,instanceid) in cluster_nodes
+        node_status = gcp_get_instance_status(cluster, instanceid)
+        @info "$nodeid ($instanceid) is $node_status"
+        error = !isnothing(cluster_status) && cluster_status != node_status 
+        cluster_status = node_status
+    end
+    if error
+        @error "The GCP cluster is in a inconsistent status (all nodes must be in the same status)"
+    else
+        @info "The cluster $cluster_handle at GCP is in $cluster_status status"
+    end
+end
+
+function cluster_delete(_::Type{GoogleCloud}, cluster_handle)
+    gcp_delete_cluster(cluster_handle)
 end