Skip to content

Commit

Permalink
CDPD-53157: [compute] Fixed warehouses sync for impala feng (#3465)
Browse files Browse the repository at this point in the history
Impala unified-analytics (feng) presents a Hive interface and should go in Hive editor rather than Impala editor.

(cherry picked from commit 40f2b2b)
(cherry picked from commit fd776e6)
Change-Id: I2c97483f858fb19783e3a5a2d78def9382e27867
(cherry picked from commit 9ded60d)
  • Loading branch information
amitsrivastava authored and wing2fly committed Nov 2, 2023
1 parent 62841b9 commit bd10f5b
Showing 1 changed file with 10 additions and 9 deletions.
19 changes: 10 additions & 9 deletions desktop/core/src/desktop/management/commands/sync_warehouses.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,10 @@ def usage(self, subcommand):


def sync_warehouses(args, options):
(hives, impalas) = get_computes_from_k8s()
computes = get_computes_from_k8s()

hives = [c for c in computes if c['dialect'] == 'hive']
impalas = [c for c in computes if c['dialect'] == 'impala']

(hive_warehouse, created) = models.Namespace.objects.get_or_create(
external_id="CDW_HIVE_WAREHOUSE",
Expand Down Expand Up @@ -89,9 +92,7 @@ def add_computes_to_warehouse(warehouse, computes):

def get_computes_from_k8s():
catalogs = []
hives = []
impalas = []
computes = {}
computes = []

for n in core_v1.list_namespace().items:
namespace = n.metadata.name
Expand All @@ -105,15 +106,13 @@ def get_computes_from_k8s():
if namespace.startswith('warehouse-'):
catalogs.append(item)
elif namespace.startswith('compute-'):
hives.append(item)
computes[namespace] = item
computes.append(item)
update_hive_configs(namespace, item, 'hiveserver2-service.%s.svc.cluster.local' % namespace)
elif namespace.startswith('impala-'):
impalas.append(item)
computes[namespace] = item
computes.append(item)
populate_impala(namespace, item)

return (hives, impalas)
return computes

def update_hive_configs(namespace, hive, host, port=80):
hs2_stfs = apps_v1.read_namespaced_stateful_set('hiveserver2', namespace)
Expand Down Expand Up @@ -204,6 +203,8 @@ def update_impala_configs(namespace, impala, host):
{"name": "hive_metastore_uris", "value": hive_metastore_uris},
]

impala.pop('server_port', None)
impala.pop('api_port', None)
impala.update({
'dialect': 'impala',
'interface': 'hiveserver2',
Expand Down

0 comments on commit bd10f5b

Please sign in to comment.