version 0.1.1 (cummulative improvements / major: load defaut CCconfig.toml when not found)

Author: decarvalhojunior-fh

Project.toml

@@ -1,7 +1,7 @@
 name = "CloudClusters"
 uuid = "4ca6f12b-c8f1-4945-b50f-6bb73234c039"
 authors = ["Francisco Heron de Carvalho Junior <heron@dc.ufc.br> e João Marcelo Uchôa de Alencar <joao.marcelo@ufc.br>"]
-version = "0.1.0"
+version = "0.1.1"
 
 [deps]
 AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"

docs/src/index.md

@@ -1,5 +1,8 @@
 ![CloudClusters.jl](https://raw.githubusercontent.com/PlatformAwareProgramming/CloudClusters.jl/refs/heads/main/docs/src/assets/logo-text.svg)
 
+[![TagBot](https://github.com/PlatformAwareProgramming/CloudClusters.jl/actions/workflows/TagBot.yml/badge.svg)](https://github.com/PlatformAwareProgramming/CloudClusters.jl/actions/workflows/TagBot.yml)
+[![CompatHelper](https://github.com/PlatformAwareProgramming/CloudClusters.jl/actions/workflows/CompatHelper.yml/badge.svg)](https://github.com/PlatformAwareProgramming/CloudClusters.jl/actions/workflows/CompatHelper.yml)
+
 _A package for creating, using, and managing clusters of virtual machine (VM) instances deployed with IaaS cloud providers._
 
 > [!NOTE]
@@ -26,15 +29,16 @@ Creating clusters with _CloudClusters.jl_ requires specifying some configuration
 * a path pointed by the CLOUD_CLUSTERS_CONFIG environment variable, if it exists;
 * the current path.
 
-Section [Configuration parameters](https://github.com/PlatformAwareProgramming/CloudClusters.jl#configuration-parameters) describes configuration parameters and how they can be overridden in programs.  
-
-Default configuration parameters can be overridden in programs. 
+Section [Configuration parameters](https://github.com/PlatformAwareProgramming/CloudClusters.jl#configuration-parameters) describes default configuration parameters and how they can be overridden in programs.  
 
 A [_CCconfig.toml_](https://raw.githubusercontent.com/PlatformAwareProgramming/CloudClusters.jl/refs/heads/main/CCconfig.toml) file is provided in the repository's top-level directory. It is configured to create clusters using prebuilt virtual machine images for each supported cloud provider. These images are based on the latest version of Ubuntu and include a Julia installation of a recent stable version with all the packages needed to instantiate the clusters added and precompiled. Users can create customized images, possibly derived from the provided image, using their preferred version of Julia and adding the packages they need. 
 
-> [!NOTE]
+> [!WARNING]
 > The version of Julia on the host computer using _CloudClusters.jl_ must be the same version as the image used to deploy the clusters.
 
+> [!NOTE]
+> The current prebuilt image for EC2 is located at the _us-east-1_ (North Virginia) region. Suppose the user is going to deploy a cluster in another region. In that case, they must create a copy of the image for that region in their account and assign their id to the ```imageid``` parameter of _CCConfig.toml_.
+
 ### The _PlatformAware.jl_ package
 
 _CloudClusters.jl_ relies on an experimental package called [_PlatformAware.jl_](https://github.com/PlatformAwareProgramming/PlatformAware.jl) for the specification of _platform types_, aimed at specifying assumptions about architectural features of virtual machines instances. Indeed, _PlatformAware.jl_ may be used with _CloudClusters.jl_ to write functions specifically tuned according to the features of VM instances that comprise the clusters. This is called _platform-aware programming_. The users of _CloudClusters.jl_, particularly package developers, are invited to explore and use the ideas behind _PlatformAware.jl_.

src/config/configs.jl

@@ -30,7 +30,6 @@ function readCCConfig(config_file::String)
             end
          end
 
-    #@info "=====> $ccconfig_toml"
     if isnothing(ccconfig_toml)
         @warn "A configuration file ($config_file) was not found. A default $config_file will be downloaded and copied to the current directory."
         fetch_default_configuration_file(config_file)
@@ -61,18 +60,15 @@ function loadDefaults(_::Type{Provider}, ccconfig_dict)
     return defaults_dict
 end
 
-_providers = [Provider, Localhost, AmazonEC2, GoogleCloud]
+_providers = [(Provider,"defaults"), (Localhost,"local"), (AmazonEC2, "ec2"), (GoogleCloud, "gcp")]
 
 defaults_dict = Dict()
 
 function load!()
     ccconfig_dict = readCCConfig("CCconfig.toml")
-    for provider_type in _providers
-        if !isnothing(ccconfig_dict)
-            defaults_dict[provider_type] = loadDefaults(provider_type, ccconfig_dict) 
-        else
-            @error "Default configuration of $provider_type is empty"
-        end
+    for (provider_type, provider_key) in _providers
+        isempty(ccconfig_dict[provider_key]) && @warn "Default configuration of $provider_type ($provider_key) is empty"
+        defaults_dict[provider_type] = loadDefaults(provider_type, ccconfig_dict) 
     end
 end
 

src/deploy.jl

@@ -63,7 +63,6 @@ function cluster_deploy(contract_handle, config_args...)
 
     if !isnothing(pids) 
         cluster_deploy_info[cluster_handle][:pids] = pids
-        @info "pids: $(cluster_deploy_info[cluster_handle][:pids])"
         return cluster_handle
     else
        @warn "error launching processes -- cluster will be terminated"
@@ -200,6 +199,8 @@ function launch_processes_ssh(cluster_features, _::Type{<:ManagerWorkers}, ips)
         ntries += 1
     end
 
+    @info "the entry process of this MW cluster has pid $(first(master_id))"
+
     return master_id
 
 end
@@ -246,6 +247,8 @@ function launch_processes_local(cluster_features, _::Type{<:ManagerWorkers}, ips
         ntries += 1
     end
 
+    @info "the entry process of this MW cluster has pid $master_id"
+
     return master_id
 
 end
@@ -298,6 +301,8 @@ function launch_processes_ssh(cluster_features, _::Type{<:PeerWorkers}, ips)
         ntries += 1
     end
 
+    @info "the worker processes of this PW cluster have pids $peer_ids"
+
     return peer_ids
 
 end
@@ -322,6 +327,8 @@ function launch_processes_local(cluster_features, _::Type{<:PeerWorkers}, ips)
         ntries += 1
     end
 
+    @info "the worker processes of this PW local cluster have pids $peer_ids"
+
     return peer_ids
 
 end
@@ -356,6 +363,8 @@ function launch_processes_mpi(cluster_features, _::Type{<:PeerWorkersMPI}, ips)
         ntries += 1
     end
 
+    @info "the worker processes of this PW-MPI cluster (local) have pids $peer_ids"
+
     return peer_ids
 
 end
@@ -436,7 +445,7 @@ function kill_processes(cluster_handle, _::Type{<:ManagerWorkers}, cluster_featu
         rmprocs(pids)
     end
     empty!(cluster_deploy_info[cluster_handle][:pids])
-    @info "pids $pids removed (manager)"
+    !isempty(pids) && @info "pids $pids removed (manager)"
 end
 
 function kill_processes(cluster_handle, _::Type{<:PeerWorkers}, cluster_features)
@@ -445,7 +454,7 @@ function kill_processes(cluster_handle, _::Type{<:PeerWorkers}, cluster_features
         rmprocs(pids)
         empty!(cluster_deploy_info[cluster_handle][:pids])
     end
-    @info "pids $pids removed (peers)"
+    !isempty(pids) && @info "pids $pids removed (peers)"
 end
 
 

src/macros.jl

@@ -7,13 +7,6 @@ function readFeatures(features)
     for f in features
         @assert f.head == :call
         @assert f.args[1] == :(=>)
-        #if arg.args[2] == :cluster_type
-        #    cluster_type = arg.args[3]
-        #    push!(common_features, Expr(:call, :(=>), :(:cluster_type), cluster_type) #= :cluster_type => cluster_type=#)
-        #elseif arg.args[2] == :node_provider
-        #    cloud_provider = arg.args[3]
-        #    push!(common_features, Expr(:call, :(=>), :(:node_provider), cloud_provider)   #=:node_provider => cloud_provider=#)
-        #else
         if isa(f.args[2], Expr) 
             @assert f.args[2].head == :.
             which_node = f.args[2].args[1]

src/resolve.jl

@@ -7,6 +7,7 @@ function cluster_resolve(contract_handle)
         cluster_type, cluster_features = cluster_contract[contract_handle]
 
         cluster_resolve(cluster_type, cluster_features, contract_handle)
+
     catch e
         println(e)
         return :fail
@@ -16,8 +17,8 @@ end
 
 function cluster_resolve(_::Type{<:ManagerWorkers}, cluster_features, contract_handle)
 
-    !haskey(cluster_features, :manager_features) && @warn ":manager_features not specified"
-    !haskey(cluster_features, :worker_features) && @warn ":worker_features not specified"
+    !haskey(cluster_features, :manager_features) && @warn ":manager_features not explicitly specified"
+    !haskey(cluster_features, :worker_features) && @warn ":worker_features not explicitly specified"
 
     manager_features = Dict{Symbol,Any}(get(cluster_features, :manager_features, cluster_features))
     worker_features = Dict{Symbol,Any}(get(cluster_features, :worker_features, cluster_features))
@@ -36,6 +37,9 @@ function cluster_resolve(_::Type{<:ManagerWorkers}, cluster_features, contract_h
 
     cluster_contract_resolved[contract_handle] = (instance_type_manager, instance_type_worker)
 
+    @info "$instance_type_manager of $node_provider selected for the manager node"
+    @info "$instance_type_worker of $node_provider selected for the worker nodes"
+
     :manager_instance_type => instance_type_manager, :worker_instance_type => instance_type_worker
 end
 
@@ -47,6 +51,8 @@ function cluster_resolve(_::Type{<:PeerWorkers}, cluster_features, contract_hand
 
     cluster_contract_resolved[contract_handle] = instance_type
 
+    @info "$instance_type of $node_provider selected for the peer nodes"
+
     :instance_type => instance_type
 end
 
@@ -65,8 +71,7 @@ function call_resolve(features)
         end
     end
 
-    str = resolve(resolve_args...)
-    return str
+    return resolve(resolve_args...)
 end
 
 #function resolve(provider::Type{<:EC2Cluster}, node_machinetype, node_memory_size, #=node_ecu_count,=# node_vcpus_count, accelerator_count, accelerator_type, accelerator_arch, accelerator, processor, processor_arch, storage_type, storage_size, interconnection_bandwidth)