Merge pull request #206 from mriccia/onlycoldstarts

Add parameter to power-tune only cold starts

Author: alexcasalboni

README-ADVANCED.md

@@ -51,13 +51,15 @@ There are three main costs associated with AWS Lambda Power Tuning:
 
 The AWS Step Functions state machine is composed of five Lambda functions:
 
-* **initializer**: create N versions and aliases corresponding to the power values provided as input (e.g. 128MB, 256MB, etc.)
-* **executor**: execute the given Lambda function `num` times, extract execution time from logs, and compute average cost per invocation
-* **cleaner**: delete all the previously generated aliases and versions
-* **analyzer**: compute the optimal power value (current logic: lowest average cost per invocation)
-* **optimizer**: automatically set the power to its optimal value (only if `autoOptimize` is `true`)
-
-Initializer, cleaner, analyzer, and optimizer are executed only once, while the executor is used by N parallel branches of the state machine - one for each configured power value. By default, the executor will execute the given Lambda function `num` consecutive times, but you can enable parallel invocation by setting `parallelInvocation` to `true`.
+* **Initializer**: define all the versions and aliases that need to be created (see Publisher below)
+* **Publisher**: create a new version and aliases corresponding to one of the power values provided as input (e.g. 128MB, 256MB, etc.)
+* **IsCountReached**: go back to Publisher until all the versiona and aliases have been created
+* **Executor**: execute the given Lambda function `num` times, extract execution time from logs, and compute average cost per invocation
+* **Cleaner**: delete all the previously generated aliases and versions
+* **Analyzer**: compute the optimal power value (current logic: lowest average cost per invocation)
+* **Optimizer**: automatically set the power to its optimal value (only if `autoOptimize` is `true`)
+
+Initializer, Cleaner, Analyzer, and Optimizer are invoked only once, while the Publisher and Executor are invoked multiple times. Publisher is used in a loop to create all the required versions and aliases, which depend on the values of `num`, `powerValues`, and `onlyColdStarts`. Executor is used by N parallel branches - one for each configured power value. By default, the Executor will invike the given Lambda function `num` consecutive times, but you can enable parallel invocation by setting `parallelInvocation` to `true`.
 
 ## Weighted Payloads
 

README.md

@@ -61,10 +61,12 @@ const findOptimalConfiguration = (event) => {
     const balancedWeight = getBalancedWeight(event);
     const optimizationFunction = optimizationStrategies[strategy]();
     const optimal = optimizationFunction(stats, balancedWeight);
+    const onlyColdStarts = event.onlyColdStarts;
+    const num = event.num;
 
     // also compute total cost of optimization state machine & lambda
     optimal.stateMachine = {};
-    optimal.stateMachine.executionCost = utils.stepFunctionsCost(event.stats.length);
+    optimal.stateMachine.executionCost = utils.stepFunctionsCost(event.stats.length, onlyColdStarts, num);
     optimal.stateMachine.lambdaCost = stats
         .map((p) => p.totalCost)
         .reduce((a, b) => a + b, 0);

imgs/state-machine-screenshot.png

@@ -8,13 +8,20 @@ const utils = require('./utils');
  */
 module.exports.handler = async(event, context) => {
 
-    const {lambdaARN, powerValues} = event;
+    const {
+        lambdaARN,
+        powerValues,
+        onlyColdStarts,
+        num,
+    } = extractDataFromInput(event);
 
     validateInput(lambdaARN, powerValues); // may throw
 
-    const ops = powerValues.map(async(value) => {
-        const alias = 'RAM' + value;
-        await cleanup(lambdaARN, alias); // may throw
+    // build list of aliases to clean up
+    const aliases = buildAliasListForCleanup(lambdaARN, onlyColdStarts, powerValues, num);
+
+    const ops = aliases.map(async(alias) => {
+        await cleanup(lambdaARN, alias);
     });
 
     // run everything in parallel and wait until completed
@@ -23,12 +30,32 @@ module.exports.handler = async(event, context) => {
     return 'OK';
 };
 
+const buildAliasListForCleanup = (lambdaARN, onlyColdStarts, powerValues, num) => {
+    if (onlyColdStarts){
+        return powerValues.map((powerValue) => {
+            return utils.range(num).map((index) => {
+                return utils.buildAliasString(`RAM${powerValue}`, onlyColdStarts, index);
+            });
+        }).flat();
+    }
+    return powerValues.map((powerValue) => utils.buildAliasString(`RAM${powerValue}`));
+};
+
+const extractDataFromInput = (event) => {
+    return {
+        lambdaARN: event.lambdaARN,
+        powerValues: event.lambdaConfigurations.powerValues,
+        onlyColdStarts: event.onlyColdStarts,
+        num: parseInt(event.num, 10), // parse as we do in the initializer
+    };
+};
+
 const validateInput = (lambdaARN, powerValues) => {
     if (!lambdaARN) {
         throw new Error('Missing or empty lambdaARN');
     }
     if (!powerValues || !powerValues.length) {
-        throw new Error('Missing or empty power values');
+        throw new Error('Missing or empty powerValues values');
     }
 };
 

lambda/analyzer.js

@@ -20,6 +20,7 @@ module.exports.handler = async(event, context) => {
         preProcessorARN,
         postProcessorARN,
         discardTopBottom,
+        onlyColdStarts,
         sleepBetweenRunsMs,
         disablePayloadLogs,
     } = await extractDataFromInput(event);
@@ -35,8 +36,11 @@ module.exports.handler = async(event, context) => {
     const lambdaAlias = 'RAM' + value;
     let results;
 
-    // fetch architecture from $LATEST
-    const {architecture, isPending} = await utils.getLambdaConfig(lambdaARN, lambdaAlias);
+    // defaulting the index to 0 as the index is required for onlyColdStarts
+    let aliasToInvoke = utils.buildAliasString(lambdaAlias, onlyColdStarts, 0);
+    // We need the architecture, regardless of onlyColdStarts or not
+    const {architecture, isPending} = await utils.getLambdaConfig(lambdaARN, aliasToInvoke);
+
     console.log(`Detected architecture type: ${architecture}, isPending: ${isPending}`);
 
     // pre-generate an array of N payloads
@@ -49,12 +53,14 @@ module.exports.handler = async(event, context) => {
         payloads: payloads,
         preARN: preProcessorARN,
         postARN: postProcessorARN,
+        onlyColdStarts: onlyColdStarts,
         sleepBetweenRunsMs: sleepBetweenRunsMs,
         disablePayloadLogs: disablePayloadLogs,
     };
 
     // wait if the function/alias state is Pending
-    if (isPending) {
+    // in the case of onlyColdStarts, we will verify each alias in the runInParallel or runInSeries
+    if (isPending && !onlyColdStarts) {
         await utils.waitForAliasActive(lambdaARN, lambdaAlias);
         console.log('Alias active');
     }
@@ -97,8 +103,14 @@ const extractDiscardTopBottomValue = (event) => {
     // extract discardTopBottom used to trim values from average duration
     let discardTopBottom = event.discardTopBottom;
     if (typeof discardTopBottom === 'undefined') {
+        // default value for discardTopBottom
         discardTopBottom = 0.2;
     }
+    // In case of onlyColdStarts, we only have 1 invocation per alias, therefore we shouldn't discard any execution
+    if (event.onlyColdStarts){
+        discardTopBottom = 0;
+        console.log('Setting discardTopBottom to 0, every invocation should be accounted when onlyColdStarts');
+    }
     // discardTopBottom must be between 0 and 0.4
     return Math.min(Math.max(discardTopBottom, 0.0), 0.4);
 };
@@ -128,16 +140,22 @@ const extractDataFromInput = async(event) => {
         preProcessorARN: input.preProcessorARN,
         postProcessorARN: input.postProcessorARN,
         discardTopBottom: discardTopBottom,
+        onlyColdStarts: !!input.onlyColdStarts,
         sleepBetweenRunsMs: sleepBetweenRunsMs,
         disablePayloadLogs: !!input.disablePayloadLogs,
     };
 };
 
-const runInParallel = async({num, lambdaARN, lambdaAlias, payloads, preARN, postARN, disablePayloadLogs}) => {
+const runInParallel = async({num, lambdaARN, lambdaAlias, payloads, preARN, postARN, disablePayloadLogs, onlyColdStarts}) => {
     const results = [];
     // run all invocations in parallel ...
     const invocations = utils.range(num).map(async(_, i) => {
-        const {invocationResults, actualPayload} = await utils.invokeLambdaWithProcessors(lambdaARN, lambdaAlias, payloads[i], preARN, postARN, disablePayloadLogs);
+        let aliasToInvoke = utils.buildAliasString(lambdaAlias, onlyColdStarts, i);
+        if (onlyColdStarts){
+            await utils.waitForAliasActive(lambdaARN, aliasToInvoke);
+            console.log(`${aliasToInvoke} is active`);
+        }
+        const {invocationResults, actualPayload} = await utils.invokeLambdaWithProcessors(lambdaARN, aliasToInvoke, payloads[i], preARN, postARN, disablePayloadLogs);
         // invocation errors return 200 and contain FunctionError and Payload
         if (invocationResults.FunctionError) {
             let errorMessage = 'Invocation error (running in parallel)';
@@ -150,11 +168,16 @@ const runInParallel = async({num, lambdaARN, lambdaAlias, payloads, preARN, post
     return results;
 };
 
-const runInSeries = async({num, lambdaARN, lambdaAlias, payloads, preARN, postARN, sleepBetweenRunsMs, disablePayloadLogs}) => {
+const runInSeries = async({num, lambdaARN, lambdaAlias, payloads, preARN, postARN, sleepBetweenRunsMs, disablePayloadLogs, onlyColdStarts}) => {
     const results = [];
     for (let i = 0; i < num; i++) {
+        let aliasToInvoke = utils.buildAliasString(lambdaAlias, onlyColdStarts, i);
         // run invocations in series
-        const {invocationResults, actualPayload} = await utils.invokeLambdaWithProcessors(lambdaARN, lambdaAlias, payloads[i], preARN, postARN, disablePayloadLogs);
+        if (onlyColdStarts){
+            await utils.waitForAliasActive(lambdaARN, aliasToInvoke);
+            console.log(`${aliasToInvoke} is active`);
+        }
+        const {invocationResults, actualPayload} = await utils.invokeLambdaWithProcessors(lambdaARN, aliasToInvoke, payloads[i], preARN, postARN, disablePayloadLogs);
         // invocation errors return 200 and contain FunctionError and Payload
         if (invocationResults.FunctionError) {
             let errorMessage = 'Invocation error (running in series)';
@@ -169,18 +192,19 @@ const runInSeries = async({num, lambdaARN, lambdaAlias, payloads, preARN, postAR
 };
 
 const computeStatistics = (baseCost, results, value, discardTopBottom) => {
-    // use results (which include logs) to compute average duration ...
-
-    const durations = utils.parseLogAndExtractDurations(results);
 
-    const averageDuration = utils.computeAverageDuration(durations, discardTopBottom);
+    // use results (which include logs) to compute average duration ...
+    const totalDurations = utils.parseLogAndExtractDurations(results);
+    const averageDuration = utils.computeAverageDuration(totalDurations, discardTopBottom);
     console.log('Average duration: ', averageDuration);
 
-    // ... and overall statistics
-    const averagePrice = utils.computePrice(baseCost, minRAM, value, averageDuration);
-
+    // ... and overall cost statistics
+    const billedDurations = utils.parseLogAndExtractBilledDurations(results);
+    const averageBilledDuration = utils.computeAverageDuration(billedDurations, discardTopBottom);
+    console.log('Average Billed duration: ', averageBilledDuration);
+    const averagePrice = utils.computePrice(baseCost, minRAM, value, averageBilledDuration);
     // .. and total cost (exact $)
-    const totalCost = utils.computeTotalCost(baseCost, minRAM, value, durations);
+    const totalCost = utils.computeTotalCost(baseCost, minRAM, value, billedDurations);
 
     const stats = {
         averagePrice,

lambda/cleaner.js

@@ -8,24 +8,58 @@ const defaultPowerValues = process.env.defaultPowerValues.split(',');
  */
 module.exports.handler = async(event, context) => {
 
-    const {lambdaARN, num} = event;
-    const powerValues = extractPowerValues(event);
+    const {
+        lambdaARN,
+        num,
+        powerValues,
+        onlyColdStarts,
+    } = extractDataFromInput(event);
 
     validateInput(lambdaARN, num); // may throw
 
     // fetch initial $LATEST value so we can reset it later
-    const initialPower = await utils.getLambdaPower(lambdaARN);
+    const {power, description} = await utils.getLambdaPower(lambdaARN);
+    console.log(power, description);
+
+    let initConfigurations = [];
 
     // reminder: configuration updates must run sequentially
     // (otherwise you get a ResourceConflictException)
-    for (let value of powerValues){
-        const alias = 'RAM' + value;
-        await utils.createPowerConfiguration(lambdaARN, value, alias);
+    for (let powerValue of powerValues){
+        const baseAlias = 'RAM' + powerValue;
+        if (!onlyColdStarts){
+            initConfigurations.push({powerValue: powerValue, alias: baseAlias});
+        } else {
+            for (let n of utils.range(num)){
+                let alias = utils.buildAliasString(baseAlias, onlyColdStarts, n);
+                // here we inject a custom description to force the creation of a new version
+                // even if the power is the same, which will force a cold start
+                initConfigurations.push({powerValue: powerValue, alias: alias, description: `${description} - ${alias}`});
+            }
+        }
     }
+    // Publish another version to revert the Lambda Function to its original configuration
+    initConfigurations.push({powerValue: power, description: description});
+
+    return {
+        initConfigurations: initConfigurations,
+        iterator: {
+            index: 0,
+            count: initConfigurations.length,
+            continue: true,
+        },
+        powerValues: powerValues,
+    };
+};
 
-    await utils.setLambdaPower(lambdaARN, initialPower);
 
-    return powerValues;
+const extractDataFromInput = (event) => {
+    return {
+        lambdaARN: event.lambdaARN,
+        num: parseInt(event.num, 10),
+        powerValues: extractPowerValues(event),
+        onlyColdStarts: !!event.onlyColdStarts,
+    };
 };
 
 const extractPowerValues = (event) => {

lambda/executor.js

@@ -0,0 +1,48 @@
+'use strict';
+
+const utils = require('./utils');
+
+
+module.exports.handler = async(event, context) => {
+    const {lambdaConfigurations, currConfig, lambdaARN} = validateInputs(event);
+    const currentIterator = lambdaConfigurations.iterator;
+    // publish version & assign alias (if present)
+    await utils.createPowerConfiguration(lambdaARN, currConfig.powerValue, currConfig.alias, currConfig.description);
+
+    const result = {
+        powerValues: lambdaConfigurations.powerValues,
+        initConfigurations: lambdaConfigurations.initConfigurations,
+        iterator: {
+            index: (currentIterator.index + 1),
+            count: currentIterator.count,
+            continue: ((currentIterator.index + 1) < currentIterator.count),
+        },
+    };
+
+    if (!result.iterator.continue) {
+        // clean the list of configuration if we're done iterating
+        delete result.initConfigurations;
+    }
+
+    return result;
+};
+function validateInputs(event) {
+    if (!event.lambdaARN) {
+        throw new Error('Missing or empty lambdaARN');
+    }
+    const lambdaARN = event.lambdaARN;
+    if (!(event.lambdaConfigurations && event.lambdaConfigurations.iterator && event.lambdaConfigurations.initConfigurations)){
+        throw new Error('Invalid iterator for initialization');
+    }
+    const iterator = event.lambdaConfigurations.iterator;
+    if (!(iterator.index >= 0 && iterator.index < iterator.count)){
+        throw new Error(`Invalid iterator index: ${iterator.index}`);
+    }
+    const lambdaConfigurations = event.lambdaConfigurations;
+    const currIdx = iterator.index;
+    const currConfig = lambdaConfigurations.initConfigurations[currIdx];
+    if (!(currConfig && currConfig.powerValue)){
+        throw new Error(`Invalid init configuration: ${JSON.stringify(currConfig)}`);
+    }
+    return {lambdaConfigurations, currConfig, lambdaARN};
+}