From e31b7d8601335f338bf810b81aad9a829f52dcf3 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 31 Jan 2025 11:50:56 -0800 Subject: [PATCH 1/2] store: Account for causality region column in InsertQuery::chunk_size The code assumed one column too few when calculating chunk size which can cause errors --- store/postgres/src/relational_queries.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index 56ad1aafacb..12c861eb352 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -2198,7 +2198,11 @@ impl<'a> InsertQuery<'a> { /// query, and depends on what columns `table` has and how they get put /// into the query pub fn chunk_size(table: &Table) -> usize { + // We always have one column for the block number/range let mut count = 1; + if table.has_causality_region { + count += 1; + } for column in table.columns.iter() { // This code depends closely on how `walk_ast` and `QueryValue` // put values into bind variables From 1b563fa8704d6f467d4a0d9366f9b078e662e6f1 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 31 Jan 2025 11:57:43 -0800 Subject: [PATCH 2/2] graph, store: Add env var GRAPH_STORE_INSERT_EXTRA_COLS --- docs/environment-variables.md | 6 ++++++ graph/src/env/store.rs | 8 ++++++++ store/postgres/src/relational_queries.rs | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 1217be769aa..5d2b501f2a6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -266,3 +266,9 @@ those. - `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION`: During the coping of a subgraph postponing creation of certain indexes (btree, attribute based ones), would speed up syncing +- `GRAPH_STORE_INSERT_EXTRA_COLS`: Makes it possible to work around bugs in + the subgraph writing code that manifest as Postgres errors saying 'number + of parameters must be between 0 and 65535' Such errors are always + graph-node bugs, but since it is hard to work around them, setting this + variable to something like 10 makes it possible to work around such a bug + while it is being fixed (default: 0) diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index ce574c94253..af0f076978f 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -128,6 +128,11 @@ pub struct EnvVarsStore { /// sufficiently, probably after 2024-12-01 /// Defaults to `false`, i.e. using the new fixed behavior pub last_rollup_from_poi: bool, + /// Safety switch to increase the number of columns used when + /// calculating the chunk size in `InsertQuery::chunk_size`. This can be + /// used to work around Postgres errors complaining 'number of + /// parameters must be between 0 and 65535' when inserting entities + pub insert_extra_cols: usize, } // This does not print any values avoid accidentally leaking any sensitive env vars @@ -177,6 +182,7 @@ impl From for EnvVarsStore { use_brin_for_all_query_types: x.use_brin_for_all_query_types, disable_block_cache_for_lookup: x.disable_block_cache_for_lookup, last_rollup_from_poi: x.last_rollup_from_poi, + insert_extra_cols: x.insert_extra_cols, } } } @@ -240,6 +246,8 @@ pub struct InnerStore { disable_block_cache_for_lookup: bool, #[envconfig(from = "GRAPH_STORE_LAST_ROLLUP_FROM_POI", default = "false")] last_rollup_from_poi: bool, + #[envconfig(from = "GRAPH_STORE_INSERT_EXTRA_COLS", default = "0")] + insert_extra_cols: usize, } #[derive(Clone, Copy, Debug)] diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index 12c861eb352..e4973649417 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -2199,7 +2199,7 @@ impl<'a> InsertQuery<'a> { /// into the query pub fn chunk_size(table: &Table) -> usize { // We always have one column for the block number/range - let mut count = 1; + let mut count = 1 + ENV_VARS.store.insert_extra_cols; if table.has_causality_region { count += 1; }