|
| 1 | +using Distributed |
| 2 | +using ArgParse |
| 3 | +using CSV |
| 4 | +using DataFrames: DataFrame |
| 5 | +using AutoAI |
| 6 | +using Statistics |
| 7 | + |
| 8 | + |
| 9 | +function parse_commandline() |
| 10 | + s = ArgParseSettings() |
| 11 | + @add_arg_table! s begin |
| 12 | + "--url", "-u" |
| 13 | + help = "mlflow server url" |
| 14 | + arg_type = String |
| 15 | + default = "http://localhost:8080" |
| 16 | + "--prediction_type", "-t" |
| 17 | + help = "classification, regression, anomalydetection" |
| 18 | + arg_type = String |
| 19 | + default = "classification" |
| 20 | + "--complexity", "-c" |
| 21 | + help = "pipeline complexity" |
| 22 | + arg_type = String |
| 23 | + default = "low" |
| 24 | + "--output_file", "-o" |
| 25 | + help = "output location" |
| 26 | + arg_type = String |
| 27 | + default = "NONE" |
| 28 | + "--nfolds", "-f" |
| 29 | + help = "number of crossvalidation folds" |
| 30 | + arg_type = Int64 |
| 31 | + default = 3 |
| 32 | + "--nworkers", "-w" |
| 33 | + help = "number of workers" |
| 34 | + arg_type = Int64 |
| 35 | + default = 5 |
| 36 | + "--no_save" |
| 37 | + help = "save model" |
| 38 | + action = :store_true |
| 39 | + "--predict_only", "-p" |
| 40 | + help = "no training, predict only" |
| 41 | + action = :store_true |
| 42 | + "--runid", "-r" |
| 43 | + help = "runid of experiment for trained model" |
| 44 | + arg_type = String |
| 45 | + default = "NONE" |
| 46 | + "csvfile" |
| 47 | + help = "input csv file" |
| 48 | + required = true |
| 49 | + end |
| 50 | + return parse_args(s; as_symbols=true) |
| 51 | +end |
| 52 | + |
| 53 | + |
| 54 | +function autoclassmode(args::Dict) |
| 55 | + url = args[:url] |
| 56 | + complexity = args[:complexity] |
| 57 | + nfolds = args[:nfolds] |
| 58 | + nworkers = args[:nworkers] |
| 59 | + prediction_type = args[:prediction_type] |
| 60 | + impl_args = (; complexity, nfolds, nworkers, prediction_type) |> pairs |> Dict |
| 61 | + fname = args[:csvfile] |
| 62 | + df = CSV.read(fname, DataFrame) |
| 63 | + X = df[:, 1:end-1] |
| 64 | + Y = df[:, end] |> collect |
| 65 | + autoclass = AutoMLFlowClassification(Dict(:url => url, :impl_args => impl_args)) |
| 66 | + Yc = fit_transform!(autoclass, X, Y) |
| 67 | + println("accuracy = ", mean(Y .== Yc)) |
| 68 | + return autoclass |
| 69 | +end |
| 70 | + |
| 71 | +function autoregmode(args::Dict) |
| 72 | + url = args[:url] |
| 73 | + complexity = args[:complexity] |
| 74 | + nfolds = args[:nfolds] |
| 75 | + nworkers = args[:nworkers] |
| 76 | + prediction_type = args[:prediction_type] |
| 77 | + impl_args = (; complexity, nfolds, nworkers, prediction_type) |> pairs |> Dict |
| 78 | + fname = args[:csvfile] |
| 79 | + df = CSV.read(fname, DataFrame) |
| 80 | + X = df[:, 1:end-1] |
| 81 | + Y = df[:, end] |> collect |
| 82 | + autoreg = AutoMLFlowRegression(Dict(:url => url, :impl_args => impl_args)) |
| 83 | + Yc = fit_transform!(autoreg, X, Y) |
| 84 | + println("mse = ", mean((Y - Yc) .^ 2)) |
| 85 | + return autoreg |
| 86 | +end |
| 87 | + |
| 88 | +function doprediction_only(args::Dict) |
| 89 | + fname = args[:csvfile] |
| 90 | + X = CSV.read(fname, DataFrame) |
| 91 | + run_id = args[:runid] |
| 92 | + url = args[:url] |
| 93 | + mlf = |
| 94 | + predtype = args[:prediction_type] |
| 95 | + mlf = if predtype == "classification" |
| 96 | + AutoMLFlowClassification(Dict(:run_id => run_id, :url => url)) |
| 97 | + elseif predtype == "regression" |
| 98 | + AutoMLFlowRegression(Dict(:run_id => run_id, :url => url)) |
| 99 | + else |
| 100 | + error("unknown predtype option") |
| 101 | + end |
| 102 | + Yn = transform!(mlf, X) |
| 103 | + ofile = args[:output_file] |
| 104 | + if ofile != "NONE" |
| 105 | + open(ofile, "w") do stfile |
| 106 | + println(stfile, "prediction: $Yn") |
| 107 | + println(stdout, "prediction: $Yn") |
| 108 | + end |
| 109 | + else |
| 110 | + println(stdout, "prediction: $Yn") |
| 111 | + end |
| 112 | + return Yn |
| 113 | +end |
| 114 | + |
| 115 | +function printsummary(io::IO, automl::Workflow) |
| 116 | + r(x) = round(x, digits=2) |
| 117 | + trainedmodel = automl.model[:automodel] |
| 118 | + bestmodel = trainedmodel.model[:bestpipeline].model[:description] |
| 119 | + println(io, "pipelines: $(trainedmodel.model[:dfpipelines].Description)") |
| 120 | + println(io, "best_pipeline: $bestmodel") |
| 121 | + bestmean = trainedmodel.model[:performance].mean[1] |
| 122 | + bestsd = trainedmodel.model[:performance].sd[1] |
| 123 | + println(io, "best_pipeline_performance: $(r(bestmean)) ± $(r(bestsd))") |
| 124 | +end |
| 125 | + |
| 126 | +function dotrainandpredict(args::Dict) |
| 127 | + # train model |
| 128 | + predtype = args[:prediction_type] |
| 129 | + automl = if predtype == "classification" |
| 130 | + autoclassmode(args) |
| 131 | + elseif predtype == "regression" |
| 132 | + autoregmode(args) |
| 133 | + end |
| 134 | + ofile = args[:output_file] |
| 135 | + if ofile != "NONE" |
| 136 | + open(ofile, "w") do stfile |
| 137 | + printsummary(stfile, automl) |
| 138 | + printsummary(stdout, automl) |
| 139 | + end |
| 140 | + else |
| 141 | + printsummary(stdout, automl) |
| 142 | + end |
| 143 | +end |
| 144 | + |
| 145 | +function (@main)(MyARGS) |
| 146 | + ARGS = parse_commandline() |
| 147 | + global _workers = ARGS[:nworkers] |
| 148 | + |
| 149 | + if ARGS[:predict_only] == false |
| 150 | + @eval (nprocs() == 1 && addprocs(_workers; exeflags=["--project=$(Base.active_project())"])) |
| 151 | + @eval (@everywhere using AutoAI) |
| 152 | + end |
| 153 | + |
| 154 | + if ARGS[:predict_only] == true |
| 155 | + # predict only using run_id of model in the artifact |
| 156 | + doprediction_only(ARGS) |
| 157 | + else |
| 158 | + # train and predict |
| 159 | + dotrainandpredict(ARGS) |
| 160 | + end |
| 161 | +end |
0 commit comments