diff --git a/cpp/cmake_modules/FindgRPCAlt.cmake b/cpp/cmake_modules/FindgRPCAlt.cmake
index 81d8cf7ca68ee..2ff10dbc23dd2 100644
--- a/cpp/cmake_modules/FindgRPCAlt.cmake
+++ b/cpp/cmake_modules/FindgRPCAlt.cmake
@@ -57,13 +57,15 @@ if(GRPCPP_PC_FOUND)
                HINTS ${GRPCPP_PC_PREFIX}
                NO_DEFAULT_PATH
                PATH_SUFFIXES "bin")
-  set(gRPCAlt_FIND_PACKAGE_ARGS gRPCAlt REQUIRED_VARS GRPCPP_IMPORTED_LOCATION
-                                GRPC_CPP_PLUGIN)
-  if(gRPCAlt_VERSION)
-    list(APPEND gRPCAlt_FIND_PACKAGE_ARGS VERSION_VAR gRPCAlt_VERSION)
-  endif()
-  find_package_handle_standard_args(${gRPCAlt_FIND_PACKAGE_ARGS})
+endif()
+set(gRPCAlt_FIND_PACKAGE_ARGS gRPCAlt REQUIRED_VARS GRPCPP_IMPORTED_LOCATION
+                              GRPC_CPP_PLUGIN)
+if(gRPCAlt_VERSION)
+  list(APPEND gRPCAlt_FIND_PACKAGE_ARGS VERSION_VAR gRPCAlt_VERSION)
+endif()
+find_package_handle_standard_args(${gRPCAlt_FIND_PACKAGE_ARGS})
 
+if(gRPCAlt_FOUND)
   # gRPC does not expose the reflection library via pkg-config, but it should be alongside the main library
   get_filename_component(GRPCPP_IMPORTED_DIRECTORY ${GRPCPP_IMPORTED_LOCATION} DIRECTORY)
   if(ARROW_GRPC_USE_SHARED)
@@ -77,11 +79,7 @@ if(GRPCPP_PC_FOUND)
                NAMES grpc++_reflection ${GRPCPP_REFLECTION_LIB_NAME}
                PATHS ${GRPCPP_IMPORTED_DIRECTORY}
                NO_DEFAULT_PATH)
-else()
-  set(gRPCAlt_FOUND FALSE)
-endif()
 
-if(gRPCAlt_FOUND)
   add_library(gRPC::grpc++ UNKNOWN IMPORTED)
   set_target_properties(gRPC::grpc++
                         PROPERTIES IMPORTED_LOCATION "${GRPCPP_IMPORTED_LOCATION}"