From 44e02e9397a9a3fc86d5c6b17e279a252be9c853 Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Wed, 17 Sep 2025 09:37:01 -0700 Subject: [PATCH 1/6] use read only pool - db cleaner Signed-off-by: Amit Prinz Setter --- src/sdk/nb.d.ts | 2 +- src/server/bg_services/db_cleaner.js | 6 +++--- src/server/object_services/md_store.js | 12 +++++++++--- src/util/postgres_client.js | 14 +++++++++++--- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index c1d14a7227..1d727113b5 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -766,7 +766,7 @@ interface DBCollection { validate(doc: object, warn?: 'warn'): object; - executeSQL(query: string, params: Array, options?: { query_name?: string }): Promise>; + executeSQL(query: string, params: Array, options?: { query_name?: string, preferred_pool?: string }): Promise>; name: any; } diff --git a/src/server/bg_services/db_cleaner.js b/src/server/bg_services/db_cleaner.js index 765ecf6ef5..60b93271a9 100644 --- a/src/server/bg_services/db_cleaner.js +++ b/src/server/bg_services/db_cleaner.js @@ -62,19 +62,19 @@ async function clean_md_store(last_date_to_remove) { } dbg.log0('DB_CLEANER: checking md-store for documents deleted before', new Date(last_date_to_remove)); const objects_to_remove = await MDStore.instance().find_deleted_objects(last_date_to_remove, config.DB_CLEANER_DOCS_LIMIT); - dbg.log2('DB_CLEANER: list objects:', objects_to_remove); + dbg.log0('DB_CLEANER: list objects:', objects_to_remove); if (objects_to_remove.length) { await P.map_with_concurrency(10, objects_to_remove, obj => db_delete_object_parts(obj)); await MDStore.instance().db_delete_objects(objects_to_remove); } const blocks_to_remove = await MDStore.instance().find_deleted_blocks(last_date_to_remove, config.DB_CLEANER_DOCS_LIMIT); - dbg.log2('DB_CLEANER: list blocks:', blocks_to_remove); + dbg.log0('DB_CLEANER: list blocks:', blocks_to_remove); if (blocks_to_remove.length) await MDStore.instance().db_delete_blocks(blocks_to_remove); const chunks_to_remove = await MDStore.instance().find_deleted_chunks(last_date_to_remove, config.DB_CLEANER_DOCS_LIMIT); const filtered_chunks = chunks_to_remove.filter(async chunk => !(await MDStore.instance().has_any_blocks_for_chunk(chunk)) && !(await MDStore.instance().has_any_parts_for_chunk(chunk))); - dbg.log2('DB_CLEANER: list chunks with no blocks and no parts to be removed from DB', filtered_chunks); + dbg.log0('DB_CLEANER: list chunks with no blocks and no parts to be removed from DB', filtered_chunks); if (filtered_chunks.length) await MDStore.instance().db_delete_chunks(filtered_chunks); dbg.log0(`DB_CLEANER: removed ${objects_to_remove.length + blocks_to_remove.length + filtered_chunks.length} documents from md-store`); } diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index 0c49cac1e7..7c2037bb49 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -1123,7 +1123,7 @@ class MDStore { FROM ${this._objects.name} WHERE (to_ts(data->>'deleted') db_client.instance().uniq_ids(objects, '_id')); } @@ -1781,6 +1782,8 @@ class MDStore { has_any_blocks_for_chunk(chunk_id) { return this._blocks.findOne({ chunk: { $eq: chunk_id, $exists: true }, + }, { + preferred_pool: 'read_only' }) .then(obj => Boolean(obj)); } @@ -1788,6 +1791,8 @@ class MDStore { has_any_parts_for_chunk(chunk_id) { return this._parts.findOne({ chunk: { $eq: chunk_id, $exists: true }, + }, { + preferred_pool: 'read_only' }) .then(obj => Boolean(obj)); } @@ -2029,7 +2034,8 @@ class MDStore { projection: { _id: 1, deleted: 1 - } + }, + preferred_pool: 'read_only' }) .then(objects => db_client.instance().uniq_ids(objects, '_id')); } diff --git a/src/util/postgres_client.js b/src/util/postgres_client.js index 993ec6970b..5c36e450e1 100644 --- a/src/util/postgres_client.js +++ b/src/util/postgres_client.js @@ -248,6 +248,9 @@ function convert_timestamps(where_clause) { async function _do_query(pg_client, q, transaction_counter) { query_counter += 1; + + dbg.log3("pg_client.options?.host =", pg_client.options?.host, ", q =", q); + const tag = `T${_.padStart(transaction_counter, 8, '0')}|Q${_.padStart(query_counter.toString(), 8, '0')}`; try { // dbg.log0(`postgres_client: ${tag}: ${q.text}`, util.inspect(q.values, { depth: 6 })); @@ -629,6 +632,10 @@ class PostgresTable { get_pool(key = this.pool_key) { const pool = this.client.get_pool(key); if (!pool) { + //if original get_pool was no for the default this.pool_key, try also this.pool_key + if (key && key !== this.pool_key) { + return this.get_pool(); + } throw new Error(`The postgres clients pool ${key} disconnected`); } return pool; @@ -716,13 +723,14 @@ class PostgresTable { * @param {Array} params * @param {{ * query_name?: string, + * preferred_pool?: string, * }} [options = {}] * * @returns {Promise>} */ async executeSQL(query, params, options = {}) { /** @type {Pool} */ - const pool = this.get_pool(); + const pool = this.get_pool(options.preferred_pool); const client = await pool.connect(); const q = { @@ -926,7 +934,7 @@ class PostgresTable { query_string += ` OFFSET ${sql_query.offset}`; } try { - const res = await this.single_query(query_string); + const res = await this.single_query(query_string, undefined, this.get_pool(options.preferred_pool)); return res.rows.map(row => decode_json(this.schema, row.data)); } catch (err) { dbg.error('find failed', query, options, query_string, err); @@ -943,7 +951,7 @@ class PostgresTable { } query_string += ' LIMIT 1'; try { - const res = await this.single_query(query_string); + const res = await this.single_query(query_string, undefined, this.get_pool(options.preferred_pool)); if (res.rowCount === 0) return null; return res.rows.map(row => decode_json(this.schema, row.data))[0]; } catch (err) { From bb1ddcc2fa00bbaae3ba60b317dbd60a913515d3 Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Wed, 24 Sep 2025 14:34:26 -0700 Subject: [PATCH 2/6] use read only pool - object reclaimer first find Signed-off-by: Amit Prinz Setter --- src/server/object_services/md_store.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index 7c2037bb49..7cbf275b9f 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -776,6 +776,7 @@ class MDStore { }, { limit: Math.min(limit, 1000), hint: 'deleted_unreclaimed_index', + preferred_pool: 'read_only', }); return results; } From 75d8db644fea889c43fe9193d9045497aa2e86d5 Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Tue, 7 Oct 2025 09:11:31 -0700 Subject: [PATCH 3/6] use read only pool - scrubber first find Signed-off-by: Amit Prinz Setter --- src/server/object_services/md_store.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index 7cbf275b9f..1e5212b16d 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -1600,6 +1600,7 @@ class MDStore { _id: -1 }, limit: limit, + preferred_pool: 'read_only', }) .then(chunks => ({ From ea626183207be47dafe7ef375af360a92d59ffae Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Mon, 20 Oct 2025 15:05:53 -0700 Subject: [PATCH 4/6] use read only pool - fix logs Signed-off-by: Amit Prinz Setter --- src/server/bg_services/db_cleaner.js | 6 +++--- src/util/postgres_client.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/bg_services/db_cleaner.js b/src/server/bg_services/db_cleaner.js index 60b93271a9..765ecf6ef5 100644 --- a/src/server/bg_services/db_cleaner.js +++ b/src/server/bg_services/db_cleaner.js @@ -62,19 +62,19 @@ async function clean_md_store(last_date_to_remove) { } dbg.log0('DB_CLEANER: checking md-store for documents deleted before', new Date(last_date_to_remove)); const objects_to_remove = await MDStore.instance().find_deleted_objects(last_date_to_remove, config.DB_CLEANER_DOCS_LIMIT); - dbg.log0('DB_CLEANER: list objects:', objects_to_remove); + dbg.log2('DB_CLEANER: list objects:', objects_to_remove); if (objects_to_remove.length) { await P.map_with_concurrency(10, objects_to_remove, obj => db_delete_object_parts(obj)); await MDStore.instance().db_delete_objects(objects_to_remove); } const blocks_to_remove = await MDStore.instance().find_deleted_blocks(last_date_to_remove, config.DB_CLEANER_DOCS_LIMIT); - dbg.log0('DB_CLEANER: list blocks:', blocks_to_remove); + dbg.log2('DB_CLEANER: list blocks:', blocks_to_remove); if (blocks_to_remove.length) await MDStore.instance().db_delete_blocks(blocks_to_remove); const chunks_to_remove = await MDStore.instance().find_deleted_chunks(last_date_to_remove, config.DB_CLEANER_DOCS_LIMIT); const filtered_chunks = chunks_to_remove.filter(async chunk => !(await MDStore.instance().has_any_blocks_for_chunk(chunk)) && !(await MDStore.instance().has_any_parts_for_chunk(chunk))); - dbg.log0('DB_CLEANER: list chunks with no blocks and no parts to be removed from DB', filtered_chunks); + dbg.log2('DB_CLEANER: list chunks with no blocks and no parts to be removed from DB', filtered_chunks); if (filtered_chunks.length) await MDStore.instance().db_delete_chunks(filtered_chunks); dbg.log0(`DB_CLEANER: removed ${objects_to_remove.length + blocks_to_remove.length + filtered_chunks.length} documents from md-store`); } diff --git a/src/util/postgres_client.js b/src/util/postgres_client.js index 5c36e450e1..e4f053a820 100644 --- a/src/util/postgres_client.js +++ b/src/util/postgres_client.js @@ -632,7 +632,7 @@ class PostgresTable { get_pool(key = this.pool_key) { const pool = this.client.get_pool(key); if (!pool) { - //if original get_pool was no for the default this.pool_key, try also this.pool_key + //if original get_pool was not for the default this.pool_key, try also this.pool_key if (key && key !== this.pool_key) { return this.get_pool(); } From 7cb9fa15efad6e3469eba245c470c7b371beb716 Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Fri, 31 Oct 2025 16:15:29 -0700 Subject: [PATCH 5/6] use read only pool - add retry with default pool Signed-off-by: Amit Prinz Setter --- src/util/postgres_client.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/util/postgres_client.js b/src/util/postgres_client.js index e4f053a820..2460141ecc 100644 --- a/src/util/postgres_client.js +++ b/src/util/postgres_client.js @@ -249,7 +249,7 @@ function convert_timestamps(where_clause) { async function _do_query(pg_client, q, transaction_counter) { query_counter += 1; - dbg.log3("pg_client.options?.host =", pg_client.options?.host, ", q =", q); + dbg.log3("pg_client.options?.host =", pg_client.options?.host, ", retry =", pg_client.retry_with_default_pool, ", q =", q); const tag = `T${_.padStart(transaction_counter, 8, '0')}|Q${_.padStart(query_counter.toString(), 8, '0')}`; try { @@ -268,6 +268,10 @@ async function _do_query(pg_client, q, transaction_counter) { if (err.routine === 'index_create' && err.code === '42P07') return; dbg.error(`postgres_client: ${tag}: failed with error:`, err); await log_query(pg_client, q, tag, 0, /*should_explain*/ false); + if (pg_client.retry_with_default_pool) { + dbg.warn("retrying with default pool. q = ", q); + return _do_query(PostgresClient.instance().get_pool('default'), q, transaction_counter); + } throw err; } } @@ -1505,7 +1509,8 @@ class PostgresClient extends EventEmitter { }, read_only: { instance: null, - size: config.POSTGRES_DEFAULT_MAX_CLIENTS + size: config.POSTGRES_DEFAULT_MAX_CLIENTS, + retry_with_default_pool: true } }; @@ -1745,6 +1750,8 @@ class PostgresClient extends EventEmitter { }; } pool.instance.on('error', pool.error_listener); + //propagate retry_with_default_pool into instance so it will be available in _do_query() + pool.instance.retry_with_default_pool = pool.retry_with_default_pool; } } From dc83dbd7b668f1e1fa0978c0ecb9e7847ebc98d0 Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Tue, 11 Nov 2025 12:02:56 -0800 Subject: [PATCH 6/6] add POSTGRES_USE_READ_ONLY flag to disable using read only replica Signed-off-by: Amit Prinz Setter --- config.js | 4 ++++ src/util/postgres_client.js | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config.js b/config.js index 1b90cf5a5c..92b19a1038 100644 --- a/config.js +++ b/config.js @@ -251,6 +251,10 @@ config.DB_TYPE = /** @type {nb.DBType} */ (process.env.DB_TYPE || 'postgres'); config.POSTGRES_DEFAULT_MAX_CLIENTS = 10; config.POSTGRES_MD_MAX_CLIENTS = (process.env.LOCAL_MD_SERVER === 'true') ? 70 : 10; +//whether to use read-only postgres replica cluster +//ro host is set by operator in process.env.POSTGRES_HOST_RO +config.POSTGRES_USE_READ_ONLY = true; + /////////////////// // SYSTEM CONFIG // /////////////////// diff --git a/src/util/postgres_client.js b/src/util/postgres_client.js index 2460141ecc..4639b7f2ee 100644 --- a/src/util/postgres_client.js +++ b/src/util/postgres_client.js @@ -1525,7 +1525,11 @@ class PostgresClient extends EventEmitter { // get the connection configuration. first from env, then from file, then default const host = process.env.POSTGRES_HOST || fs_utils.try_read_file_sync(process.env.POSTGRES_HOST_PATH) || '127.0.0.1'; //optional read-only host. if not present defaults to general pg host - const host_ro = process.env.POSTGRES_HOST_RO || fs_utils.try_read_file_sync(process.env.POSTGRES_HOST_RO_PATH) || host; + let host_ro = process.env.POSTGRES_HOST_RO || fs_utils.try_read_file_sync(process.env.POSTGRES_HOST_RO_PATH) || host; + //if POSTGRES_USE_READ_ONLY is off, switch to regular host + if (!config.POSTGRES_USE_READ_ONLY) { + host_ro = host; + } const user = process.env.POSTGRES_USER || fs_utils.try_read_file_sync(process.env.POSTGRES_USER_PATH) || 'postgres'; const password = process.env.POSTGRES_PASSWORD || fs_utils.try_read_file_sync(process.env.POSTGRES_PASSWORD_PATH) || 'noobaa'; const database = process.env.POSTGRES_DBNAME || fs_utils.try_read_file_sync(process.env.POSTGRES_DBNAME_PATH) || 'nbcore';