feat: Respect the package-lock.json for a NodeJS Lambda function (#423)

Respect the `package-lock.json` so NodeJS Lambda for reproducible builds
which are critical in production environments. Similarly like for the
Poetry, copy a lock file, if such is present, to a temporary build
directory. npm will use a `package-lock.json` file when available in
a working directory.

In the example `package.json`, require lower `requests` version to
demonstrate `package-lock.json` usage. `package.json` specifies
`~0.2.0` and the latest available matching version is `0.2.2`, but
`package-lock.json` freezes version `0.2.1` and that version gets
installed with this change, while previously the `0.2.2` would be
installed.

Author: Amadeusz Żołnowski

examples/build-package/README.md

@@ -45,6 +45,7 @@ Note that this example may create resources which cost money. Run `terraform des
 | <a name="module_package_dir_poetry"></a> [package\_dir\_poetry](#module\_package\_dir\_poetry) | ../../ | n/a |
 | <a name="module_package_dir_poetry_no_docker"></a> [package\_dir\_poetry\_no\_docker](#module\_package\_dir\_poetry\_no\_docker) | ../../ | n/a |
 | <a name="module_package_dir_with_npm_install"></a> [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a |
+| <a name="module_package_dir_with_npm_install_lock_file"></a> [package\_dir\_with\_npm\_install\_lock\_file](#module\_package\_dir\_with\_npm\_install\_lock\_file) | ../../ | n/a |
 | <a name="module_package_dir_without_npm_install"></a> [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a |
 | <a name="module_package_dir_without_pip_install"></a> [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a |
 | <a name="module_package_file"></a> [package\_file](#module\_package\_file) | ../../ | n/a |
@@ -53,6 +54,7 @@ Note that this example may create resources which cost money. Run `terraform des
 | <a name="module_package_src_poetry2"></a> [package\_src\_poetry2](#module\_package\_src\_poetry2) | ../../ | n/a |
 | <a name="module_package_with_commands_and_patterns"></a> [package\_with\_commands\_and\_patterns](#module\_package\_with\_commands\_and\_patterns) | ../../ | n/a |
 | <a name="module_package_with_docker"></a> [package\_with\_docker](#module\_package\_with\_docker) | ../../ | n/a |
+| <a name="module_package_with_npm_lock_in_docker"></a> [package\_with\_npm\_lock\_in\_docker](#module\_package\_with\_npm\_lock\_in\_docker) | ../../ | n/a |
 | <a name="module_package_with_npm_requirements_in_docker"></a> [package\_with\_npm\_requirements\_in\_docker](#module\_package\_with\_npm\_requirements\_in\_docker) | ../../ | n/a |
 | <a name="module_package_with_patterns"></a> [package\_with\_patterns](#module\_package\_with\_patterns) | ../../ | n/a |
 | <a name="module_package_with_pip_requirements_in_docker"></a> [package\_with\_pip\_requirements\_in\_docker](#module\_package\_with\_pip\_requirements\_in\_docker) | ../../ | n/a |

examples/build-package/main.tf

@@ -365,6 +365,18 @@ module "package_dir_with_npm_install" {
   source_path = "${path.module}/../fixtures/nodejs14.x-app1"
 }
 
+# Create zip-archive of a single directory where "npm install" will also be
+# executed (default for nodejs runtime). This example has package-lock.json which
+# is respected when installing dependencies.
+module "package_dir_with_npm_install_lock_file" {
+  source = "../../"
+
+  create_function = false
+
+  runtime     = "nodejs14.x"
+  source_path = "${path.module}/../fixtures/nodejs14.x-app2"
+}
+
 # Create zip-archive of a single directory without running "npm install" (which is the default for nodejs runtime)
 module "package_dir_without_npm_install" {
   source = "../../"
@@ -393,6 +405,20 @@ module "package_with_npm_requirements_in_docker" {
   hash_extra      = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install"
 }
 
+# Create zip-archive of a single directory where "npm install" will also be
+# executed using docker. This example has package-lock.json which is respected
+# when installing dependencies.
+module "package_with_npm_lock_in_docker" {
+  source = "../../"
+
+  create_function = false
+
+  runtime         = "nodejs14.x"
+  source_path     = "${path.module}/../fixtures/nodejs14.x-app2"
+  build_in_docker = true
+  hash_extra      = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install"
+}
+
 ################################
 # Build package in Docker and
 # use it to deploy Lambda Layer

examples/fixtures/nodejs14.x-app2/index.js

@@ -0,0 +1,16 @@
+'use strict';
+
+module.exports.hello = async (event) => {
+  console.log(event);
+  return {
+    statusCode: 200,
+    body: JSON.stringify(
+      {
+        message: `Go Serverless.tf! Your Nodejs function executed successfully!`,
+        input: event,
+      },
+      null,
+      2
+    ),
+  };
+};

examples/fixtures/nodejs14.x-app2/package-lock.json

@@ -0,0 +1,8 @@
+{
+  "name": "nodejs14.x-app1",
+  "version": "1.0.0",
+  "main": "index.js",
+  "dependencies": {
+    "requests": "^0.2.0"
+  }
+}

examples/fixtures/nodejs14.x-app2/package.json

@@ -733,6 +733,14 @@ def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None):
             requirements = path
             if os.path.isdir(path):
                 requirements = os.path.join(path, "package.json")
+                npm_lock_file = os.path.join(path, "package-lock.json")
+            else:
+                npm_lock_file = os.path.join(os.path.dirname(path), "package-lock.json")
+
+            if os.path.isfile(npm_lock_file):
+                hash(npm_lock_file)
+                log.info("Added npm lock file: %s", npm_lock_file)
+
             if not os.path.isfile(requirements):
                 if required:
                     raise RuntimeError("File not found: {}".format(requirements))
@@ -1395,9 +1403,10 @@ def install_npm_requirements(query, requirements_file, tmp_dir):
 
     log.info("Installing npm requirements: %s", requirements_file)
     with tempdir(tmp_dir) as temp_dir:
-        requirements_filename = os.path.basename(requirements_file)
-        target_file = os.path.join(temp_dir, requirements_filename)
-        shutil.copyfile(requirements_file, target_file)
+        temp_copy = TemporaryCopy(os.path.dirname(requirements_file), temp_dir, log)
+        temp_copy.add(os.path.basename(requirements_file))
+        temp_copy.add("package-lock.json", required=False)
+        temp_copy.copy_to_target_dir()
 
         subproc_env = None
         npm_exec = "npm"
@@ -1442,10 +1451,63 @@ def install_npm_requirements(query, requirements_file, tmp_dir):
                         "available in system PATH".format(runtime)
                     ) from e
 
-            os.remove(target_file)
+            temp_copy.remove_from_target_dir()
             yield temp_dir
 
 
+class TemporaryCopy:
+    """Temporarily copy files to a specified location and remove them when
+    not needed.
+    """
+
+    def __init__(self, source_dir_path, target_dir_path, logger=None):
+        """Initialise with a target and a source directories."""
+        self.source_dir_path = source_dir_path
+        self.target_dir_path = target_dir_path
+        self._filenames = []
+        self._logger = logger
+
+    def _make_source_path(self, filename):
+        return os.path.join(self.source_dir_path, filename)
+
+    def _make_target_path(self, filename):
+        return os.path.join(self.target_dir_path, filename)
+
+    def add(self, filename, *, required=True):
+        """Add a file to be copied from from source to target directory
+        when `TemporaryCopy.copy_to_target_dir()` is called.
+
+        By default, the file must exist in the source directory. Set `required`
+        to `False` if the file is optional.
+        """
+        if os.path.exists(self._make_source_path(filename)):
+            self._filenames.append(filename)
+        elif required:
+            raise RuntimeError("File not found: {}".format(filename))
+
+    def copy_to_target_dir(self):
+        """Copy files (added so far) to the target directory."""
+        for filename in self._filenames:
+            if self._logger:
+                self._logger.info("Copying temporarily '%s'", filename)
+
+            shutil.copyfile(
+                self._make_source_path(filename),
+                self._make_target_path(filename),
+            )
+
+    def remove_from_target_dir(self):
+        """Remove files (added so far) from the target directory."""
+        for filename in self._filenames:
+            if self._logger:
+                self._logger.info("Removing temporarily copied '%s'", filename)
+
+            try:
+                os.remove(self._make_target_path(filename))
+            except FileNotFoundError:
+                pass
+
+
 def docker_image_id_command(tag):
     """"""
     docker_cmd = ["docker", "images", "--format={{.ID}}", tag]

package.py

@@ -4,8 +4,8 @@ module "wrapper" {
   for_each = var.items
 
   build_args                = try(each.value.build_args, var.defaults.build_args, {})
-  build_target              = try(each.value.build_target, var.defaults.build_target, null)
   builder                   = try(each.value.builder, var.defaults.builder, null)
+  build_target              = try(each.value.build_target, var.defaults.build_target, null)
   cache_from                = try(each.value.cache_from, var.defaults.cache_from, [])
   create_ecr_repo           = try(each.value.create_ecr_repo, var.defaults.create_ecr_repo, false)
   create_sam_metadata       = try(each.value.create_sam_metadata, var.defaults.create_sam_metadata, false)

wrappers/docker-build/main.tf

@@ -75,8 +75,8 @@ module "wrapper" {
   lambda_at_edge_logs_all_regions              = try(each.value.lambda_at_edge_logs_all_regions, var.defaults.lambda_at_edge_logs_all_regions, true)
   lambda_role                                  = try(each.value.lambda_role, var.defaults.lambda_role, "")
   layer_name                                   = try(each.value.layer_name, var.defaults.layer_name, "")
-  layer_skip_destroy                           = try(each.value.layer_skip_destroy, var.defaults.layer_skip_destroy, false)
   layers                                       = try(each.value.layers, var.defaults.layers, null)
+  layer_skip_destroy                           = try(each.value.layer_skip_destroy, var.defaults.layer_skip_destroy, false)
   license_info                                 = try(each.value.license_info, var.defaults.license_info, "")
   local_existing_package                       = try(each.value.local_existing_package, var.defaults.local_existing_package, null)
   logging_application_log_level                = try(each.value.logging_application_log_level, var.defaults.logging_application_log_level, "INFO")
@@ -101,8 +101,8 @@ module "wrapper" {
   putin_khuylo                                 = try(each.value.putin_khuylo, var.defaults.putin_khuylo, true)
   recreate_missing_package                     = try(each.value.recreate_missing_package, var.defaults.recreate_missing_package, true)
   recursive_loop                               = try(each.value.recursive_loop, var.defaults.recursive_loop, null)
-  replace_security_groups_on_destroy           = try(each.value.replace_security_groups_on_destroy, var.defaults.replace_security_groups_on_destroy, null)
   replacement_security_group_ids               = try(each.value.replacement_security_group_ids, var.defaults.replacement_security_group_ids, null)
+  replace_security_groups_on_destroy           = try(each.value.replace_security_groups_on_destroy, var.defaults.replace_security_groups_on_destroy, null)
   reserved_concurrent_executions               = try(each.value.reserved_concurrent_executions, var.defaults.reserved_concurrent_executions, -1)
   role_description                             = try(each.value.role_description, var.defaults.role_description, null)
   role_force_detach_policies                   = try(each.value.role_force_detach_policies, var.defaults.role_force_detach_policies, true)