From 5b92afb3393801ed535e5ebc6a836bce23c03339 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Wed, 14 Apr 2021 19:01:15 -0500 Subject: [PATCH 1/4] Fix for issue where git-secrets could fail on Windows if enough files where modified to be over the 32727 command line length When git-secrets is run, for all filepaths that are part of the staged list of changes they get passed into an invocation of `git grep` If there is a large number of staged files, depending on the length of each file path from the git repo root, it could result in an extremely long line being supplied to `git grep` For example if the git structure is as below ``` / Include/ TestLibrary/ MyTestComponent1.h MyTestComponent2.h ... MyTestComponent1000.h ``` It would take around changes to 900 "Include/TestLibrary/MyTestComponentNNNN.h` files within the repo to cause the `git grep` to be passed an argument list that is too long --- git-secrets | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git-secrets b/git-secrets index 11be153..bc0f7ea 100755 --- a/git-secrets +++ b/git-secrets @@ -113,7 +113,7 @@ git_grep() { local files=("${@}") combined_patterns=$(load_combined_patterns) [ -z "${combined_patterns}" ] && return 1 - GREP_OPTIONS= LC_ALL=C git grep -nwHEI ${options} "${combined_patterns}" -- "${files[@]}" + GREP_OPTIONS= LC_ALL=C echo "${files[@]}" | xargs git grep -nwHEI ${options} "${combined_patterns}" } # Performs a regular grep, taking into account patterns and recursion. @@ -139,7 +139,9 @@ process_output() { && return 1 || return 0 ;; 1) return 0 ;; - *) exit $status + *) + echo "${output}" >&2 + exit $status esac } From e0e010379d221be0921980285daef188647e2e47 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Wed, 14 Apr 2021 20:01:22 -0500 Subject: [PATCH 2/4] Mapped the return code of git_grep to 1 if xargs returns 123 --- git-secrets | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git-secrets b/git-secrets index bc0f7ea..94232ce 100755 --- a/git-secrets +++ b/git-secrets @@ -114,6 +114,10 @@ git_grep() { [ -z "${combined_patterns}" ] && return 1 GREP_OPTIONS= LC_ALL=C echo "${files[@]}" | xargs git grep -nwHEI ${options} "${combined_patterns}" + rc=$? + [ $rc -eq 123 ] && return 1 # xargs returns 123 when the invocations return a value between 1-125 + [ $rc -eq 0 ] && return 0 + return $rc } # Performs a regular grep, taking into account patterns and recursion. From ed6aeca7288b08703b12480c93af650718c6d8cf Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Wed, 14 Apr 2021 20:23:16 -0500 Subject: [PATCH 3/4] Separated each filename passed to xargs with the NUL character of \0 This allows filenames with spaces to be properly parsed as a single file when invoking `git grep` --- git-secrets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-secrets b/git-secrets index 94232ce..c49e049 100755 --- a/git-secrets +++ b/git-secrets @@ -113,7 +113,7 @@ git_grep() { local files=("${@}") combined_patterns=$(load_combined_patterns) [ -z "${combined_patterns}" ] && return 1 - GREP_OPTIONS= LC_ALL=C echo "${files[@]}" | xargs git grep -nwHEI ${options} "${combined_patterns}" + GREP_OPTIONS= LC_ALL=C printf "%s\0" "${files[@]}" | xargs -0 git grep -nwHEI ${options} "${combined_patterns}" rc=$? [ $rc -eq 123 ] && return 1 # xargs returns 123 when the invocations return a value between 1-125 [ $rc -eq 0 ] && return 0 From 5d2073f2259394806dd63b2875713177e5ba50b7 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Wed, 14 Apr 2021 20:36:55 -0500 Subject: [PATCH 4/4] If the array of files is empty don't use xargs --- git-secrets | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git-secrets b/git-secrets index c49e049..16b6fe2 100755 --- a/git-secrets +++ b/git-secrets @@ -113,7 +113,12 @@ git_grep() { local files=("${@}") combined_patterns=$(load_combined_patterns) [ -z "${combined_patterns}" ] && return 1 - GREP_OPTIONS= LC_ALL=C printf "%s\0" "${files[@]}" | xargs -0 git grep -nwHEI ${options} "${combined_patterns}" + if (( ${#files[@]} )); then + GREP_OPTIONS= LC_ALL=C printf "%s\0" "${files[@]}" | xargs -0 git grep -nwHEI ${options} "${combined_patterns}" + else + GREP_OPTIONS= LC_ALL=C git grep -nwHEI ${options} "${combined_patterns}" + fi + rc=$? [ $rc -eq 123 ] && return 1 # xargs returns 123 when the invocations return a value between 1-125 [ $rc -eq 0 ] && return 0