Merge pull request #9267 from naveenpaul1/bucket_arn_upgrade

IAM | Bucket policy Principal upgrade script

Author: naveenpaul1

src/endpoint/s3/s3_rest.js

@@ -237,6 +237,7 @@ async function authorize_request_policy(req) {
         public_access_block,
     } = await req.object_sdk.read_bucket_sdk_policy_info(req.params.bucket);
 
+    dbg.log2('authorize_request_policy:', { bucket_owner, owner_account });
     const auth_token = req.object_sdk.get_auth_token();
     const arn_path = _get_arn_from_req_path(req);
     const method = _get_method_from_req(req);

src/test/integration_tests/internal/test_upgrade_scripts.js

@@ -4,13 +4,15 @@
 // setup coretest first to prepare the env
 const _ = require('lodash');
 const coretest = require('../../utils/coretest/coretest');
-const { rpc_client, EMAIL } = coretest;
+const { rpc_client, EMAIL, POOL_LIST } = coretest;
 coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] });
 const { S3 } = require('@aws-sdk/client-s3');
 const { NodeHttpHandler } = require("@smithy/node-http-handler");
 const http = require('http');
+const SensitiveString = require('../../../util/sensitive_string');
 const system_store = require('../../../server/system_services/system_store').get_instance();
 const upgrade_bucket_policy = require('../../../upgrade/upgrade_scripts/5.15.6/upgrade_bucket_policy');
+const upgrade_bucket_policy_principal = require('../../../upgrade/upgrade_scripts/5.21.0/upgrade_bucket_policy_principal');
 const upgrade_bucket_cors = require('../../../upgrade/upgrade_scripts/5.19.0/upgrade_bucket_cors');
 const remove_mongo_pool = require('../../../upgrade/upgrade_scripts/5.20.0/remove_mongo_pool');
 const dbg = require('../../../util/debug_module')(__filename);
@@ -20,6 +22,7 @@ const config = require('../../../../config');
 
 const BKT = 'test-bucket';
 const BKT1 = 'test-bucket1';
+const iam_username = 'iam_username';
 /** @type {S3} */
 let s3;
 
@@ -199,8 +202,85 @@ mocha.describe('test upgrade scripts', async function() {
         assert.strictEqual(updated_bucket.tiering.tiers[0].tier.mirrors[0].spread_pools[0].name, default_pool_name);
     });
 
+    mocha.it('test upgrade bucket policy to ARN version 5.21.0', async function() {
+        const old_policy = {
+            Version: '2012-10-17',
+            Statement: [{
+                    Sid: 'id-1',
+                    Effect: 'Allow',
+                    Principal: {
+                        "AWS": [new SensitiveString(EMAIL)],
+                    },
+                    Action: ['s3:GetObject', 's3:*'],
+                    Resource: [`arn:aws:s3:::*`]
+                },
+                {
+                    Effect: 'Deny',
+                    Principal: {
+                        "AWS": [new SensitiveString(iam_username)],
+                    },
+                    Action: ['s3:PutObject'],
+                    Resource: [`arn:aws:s3:::*`]
+                },
+            ]
+        };
+        // clean all leftover bucket policies as upgrade script doesn't work on updated policies 
+        await _clean_all_bucket_policies();
+
+        const bucket = system_store.data.buckets.find(bucket_obj => bucket_obj.name.unwrap() === BKT);
+        await system_store.make_changes({
+            update: {
+                buckets: [{
+                    _id: bucket._id,
+                    s3_policy: old_policy
+                }]
+            }
+        });
+        const account = system_store.data.accounts.find(acc => acc.email.unwrap() === EMAIL);
+        const nsr = 's3_bucket_policy_nsr';
+        const iam_acc = {
+                name: iam_username,
+                email: iam_username,
+                has_login: false,
+                s3_access: true,
+                default_resource: process.env.NC_CORETEST ? nsr : POOL_LIST[1].name,
+            };
+        await rpc_client.account.create_account(iam_acc);
+
+        const iam_account = system_store.data.accounts.find(acc => acc.email.unwrap() === iam_username);
+        await system_store.make_changes({
+            update: {
+                accounts: [{
+                    _id: iam_account._id,
+                    owner: account._id.toString(),
+                }]
+            }
+        });
+
+        await upgrade_bucket_policy_principal.run({ dbg, system_store, system_server: null });
+        const res = await s3.getBucketPolicy({ // should work - bucket policy should fit current schema
+            Bucket: BKT,
+        });
+        const new_policy = JSON.parse(res.Policy);
+
+        assert.strictEqual(new_policy.Statement.length, old_policy.Statement.length);
+        assert.strictEqual(new_policy.Version, old_policy.Version);
+        assert.strictEqual(new_policy.Statement[0].Sid, old_policy.Statement[0].Sid);
+        assert.strictEqual(new_policy.Statement[0].Effect, 'Allow');
+        assert.strictEqual(new_policy.Statement[0].Action[0], 's3:GetObject');
+        assert.strictEqual(new_policy.Statement[0].Action[1], 's3:*');
+        assert.strictEqual(new_policy.Statement[0].Resource[0], old_policy.Statement[0].Resource[0]);
+
+        assert.strictEqual(new_policy.Statement[0].Principal.AWS[0], `arn:aws:iam::${account._id.toString()}:root`);
+        assert.strictEqual(new_policy.Statement[1].Principal.AWS[0], `arn:aws:iam::${iam_account._id.toString()}:user/${iam_account.email.unwrap()}`);
+    });
+
     mocha.after(async function() {
         await s3.deleteBucket({ Bucket: BKT });
         await s3.deleteBucket({ Bucket: BKT1 });
+        const iam_acc = {
+                email: iam_username,
+        };
+        await rpc_client.account.delete_account(iam_acc);
     });
 });

src/upgrade/upgrade_scripts/5.21.0/upgrade_bucket_policy_principal.js

@@ -0,0 +1,65 @@
+/* Copyright (C) 2023 NooBaa */
+"use strict";
+
+const iam_constants = require('../../../endpoint/iam/iam_constants');
+
+function _create_arn(dbg, principals, system_store) {
+    if (!principals.AWS) return;
+    const principal_arns = [];
+    for (const principal of principals.AWS) {
+        if (principal === '*') continue;
+        const account = system_store.data.accounts.find(acc => acc.email.unwrap() === principal.unwrap());
+        if (!account) {
+            dbg.log0(`Could not find the account with email: ${principal}`);
+            continue;
+        }
+        let arn;
+        if (account.owner) {
+            const iam_path = account.iam_path || iam_constants.IAM_DEFAULT_PATH;
+            arn = `arn:aws:iam::${account._id.toString()}:user${iam_path}${account.email.unwrap()}`;
+        } else {
+            arn = `arn:aws:iam::${account._id.toString()}:root`;
+        }
+        principal_arns.push(arn);
+    }
+    return { AWS: principal_arns };
+}
+
+async function run({ dbg, system_store, system_server }) {
+    try {
+        dbg.log0('Starting bucket policy Principal upgrade...');
+        const buckets = [];
+        for (const bucket of system_store.data.buckets) {
+            // Do not update if there are no bucket policy.
+            if (!bucket.s3_policy) continue;
+            if (bucket.s3_policy.Statement !== undefined) {
+                const new_policy = bucket.s3_policy;
+                new_policy.Statement = bucket.s3_policy.Statement.map(statement => ({
+                    ...statement,
+                    Principal: _create_arn(dbg, statement.Principal, system_store),
+                }));
+                buckets.push({
+                    _id: bucket._id,
+                    s3_policy: new_policy,
+                });
+            }
+        }
+
+        if (buckets.length > 0) {
+            dbg.log0(`Replacing bucket policy Principal for ${buckets.length} buckets.`);
+            await system_store.make_changes({ update: { buckets } });
+        } else {
+            dbg.log0('Upgrading buckets policy Principal: no upgrade needed...');
+        }
+
+    } catch (err) {
+        dbg.error('Got error while upgrading buckets policy Principal:', err);
+        throw err;
+    }
+}
+
+
+module.exports = {
+    run,
+    description: 'Update bucket policy Principal to ARN format'
+};