V2.9.0 (#197)

_cmd.py

@@ -38,9 +38,9 @@
 import _environ as ENV
 from ssh import LocalClient
 from const import (
-    CONST_OBD_HOME, CONST_OBD_INSTALL_PATH, CONST_OBD_INSTALL_PRE, 
+    CONST_OBD_HOME,
     VERSION, REVISION, BUILD_BRANCH, BUILD_TIME, FORBIDDEN_VARS,
-    COMP_OCEANBASE_DIAGNOSTIC_TOOL
+    COMP_OCEANBASE_DIAGNOSTIC_TOOL, PKG_RPM_FILE,PKG_REPO_FILE
 )
 
 
@@ -186,8 +186,8 @@ def _mk_usage(self):
 class ObdCommand(BaseCommand):
 
     OBD_PATH = OBD_HOME_PATH
-    OBD_INSTALL_PRE = os.environ.get(CONST_OBD_INSTALL_PRE, '/')
-    OBD_INSTALL_PATH = os.environ.get(CONST_OBD_INSTALL_PATH, os.path.join(OBD_INSTALL_PRE, 'usr/obd/'))
+    OBD_INSTALL_PRE = COMMAND_ENV.get(ENV.ENV_OBD_INSTALL_PRE, '/')
+    OBD_INSTALL_PATH = COMMAND_ENV.get(ENV.ENV_OBD_INSTALL_PATH, os.path.join(OBD_INSTALL_PRE, 'usr/obd/'))
 
     def init_home(self):
         version_path = os.path.join(self.OBD_PATH, 'version')
@@ -612,7 +612,23 @@ def __init__(self):
     def _do_command(self, obd):
         url = self.cmds[0]
         return obd.mirror_manager.add_repo(url)
-
+
+
+class MirrorCleanPkgCommand(ObdCommand):
+
+    def __init__(self):
+        super(MirrorCleanPkgCommand, self).__init__('clean', 'After the list of files to be deleted is displayed, double confirm and then clean up them.')
+        self.parser.add_option('-y', '--confirm', action='store_true', help="confirm to clean up.")
+        self.parser.add_option('-c', '--components', type='string',  help="Clean up specified components. Separate multiple components with `,`.")
+        self.parser.add_option('-t', '--type', type='string', help="Specify the file types to be deleted as '%s or %s'." % (PKG_RPM_FILE, PKG_REPO_FILE))
+        self.parser.add_option('--hash', type='string', help="Repository's md5")
+
+    def _do_command(self, obd):
+        if self.opts.type and self.opts.type not in [PKG_RPM_FILE, PKG_REPO_FILE]:
+            ROOT_IO.error("Invalid type specified. Please specify '%s' or '%s'." % (PKG_RPM_FILE, PKG_REPO_FILE))
+            return False
+        return obd.clean_pkg(self.opts)
+
 
 class MirrorMajorCommand(MajorCommand):
 
@@ -625,6 +641,7 @@ def __init__(self):
         self.register_command(MirrorEnableCommand())
         self.register_command(MirrorDisableCommand())
         self.register_command(MirrorAddRepoCommand())
+        self.register_command(MirrorCleanPkgCommand())
 
 
 class RepositoryListCommand(ObdCommand):
@@ -750,7 +767,7 @@ class DemoCommand(ClusterMirrorCommand):
 
     def __init__(self):
         super(DemoCommand, self).__init__('demo', 'Quickly start')
-        self.parser.add_option('-c', '--components', type='string', help="List the components. Multiple components are separated with commas. [oceanbase-ce,obproxy-ce,obagent,prometheus,grafana]\nExample: \nstart oceanbase-ce: obd demo -c oceanbase-ce\n"
+        self.parser.add_option('-c', '--components', type='string', help="List the components. Multiple components are separated with commas. [oceanbase-ce,obproxy-ce,obagent,prometheus,grafana,ob-configserver]\nExample: \nstart oceanbase-ce: obd demo -c oceanbase-ce\n"
          + "start -c oceanbase-ce V3.2.3: obd demo -c oceanbase-ce --oceanbase-ce.version=3.2.3\n"
          + "start oceanbase-ce and obproxy-ce: obd demo -c oceanbase-ce,obproxy-ce", default='oceanbase-ce,obproxy-ce,obagent,prometheus,grafana')
         self.parser.allow_undefine = True
@@ -1113,6 +1130,7 @@ def __init__(self):
         self.parser.add_option('--primary-zone', type='string', help="Tenant primary zone. [RANDOM].", default='RANDOM')
         self.parser.add_option('--locality', type='string', help="Tenant locality.")
         self.parser.add_option('-s', '--variables', type='string', help="Set the variables for the system tenant. [ob_tcp_invited_nodes='%'].", default="ob_tcp_invited_nodes='%'")
+        self.parser.add_option('-o', '--optimize', type='string', help="Specify scenario optimization when creating a tenant, the default is consistent with the cluster dimension.\n{express_oltp, complex_oltp, olap, htap, kv}\nSupported since version 4.3.")
 
     def _do_command(self, obd):
         if len(self.cmds) == 1:
@@ -1237,6 +1255,24 @@ def _do_command(self, obd):
             return self._show_help()
 
 
+class ClusterTenantOptimizeCommand(ClusterMirrorCommand):
+
+    def __init__(self):
+        super(ClusterTenantOptimizeCommand, self).__init__('optimize','Optimizing existing tenant scenarios')
+        self.parser.add_option('-o', '--optimize', type='string', help='Optimize scenarios,the default is consistent with the cluster dimension.\n{express_oltp, complex_oltp, olap, htap, kv}\nSupported since version 4.3.')
+
+    def init(self, cmd, args):
+        super(ClusterTenantOptimizeCommand, self).init(cmd, args)
+        self.parser.set_usage('%s <deploy name> <tenant name> [options]' % self.prev_cmd)
+        return self
+
+    def _do_command(self, obd):
+        if len(self.cmds) == 2:
+            return obd.tenant_optimize(self.cmds[0], self.cmds[1])
+        else:
+            return self._show_help()
+
+
 class ClusterTenantCommand(MajorCommand):
 
     def __init__(self):
@@ -1248,6 +1284,7 @@ def __init__(self):
         self.register_command(ClusterTenantSwitchoverCommand())
         self.register_command(ClusterTenantFailoverCommand())
         self.register_command(ClusterTenantDecoupleCommand())
+        self.register_command(ClusterTenantOptimizeCommand())
 
 
 class ClusterMajorCommand(MajorCommand):
@@ -1698,6 +1735,7 @@ def __init__(self):
         self.register_command(ObdiagGatherPlanMonitorCommand())
         self.register_command(ObdiagGatherObproxyLogCommand())
         self.register_command(ObdiagGatherSceneCommand())
+        self.register_command(ObdiagGatherAshReportCommand())
 
 
 class ObdiagGatherSceneCommand(MajorCommand):
@@ -1862,7 +1900,7 @@ def __init__(self):
         super(ObdiagGatherPlanMonitorCommand, self).__init__('plan_monitor', 'Gather ParalleSQL information')
         self.parser.add_option('--trace_id', type='string', help='sql trace id')
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
-        self.parser.add_option('--env', type='string', help='env, eg: "{env1=xxx, env2=xxx}"')
+        self.parser.add_option('--env', type='string', help='''env, eg: "{db_connect='-h127.0.0.1 -P2881 -utest@test -p****** -Dtest'}"''')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
 
@@ -1881,7 +1919,7 @@ def __init__(self):
         self.parser.add_option('--from', type='string', help="specify the start of the time range. format: yyyy-mm-dd hh:mm:ss")
         self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss")
         self.parser.add_option('--since', type='string',  help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: <n> <m|h|d>. example: 1h.",default='30m')
-        self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]",default='all')
+        self.parser.add_option('--scope', type='string', help="log type constrains, choices=[obproxy, obproxy_limit, obproxy_stat, obproxy_digest, obproxy_slow, obproxy_diagnosis, obproxy_error, all]", default='all')
         self.parser.add_option('--grep', type='string', help="specify keywords constrain")
         self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false")
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
@@ -1915,7 +1953,6 @@ def __init__(self):
         self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss")
         self.parser.add_option('--since', type='string',  help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: <n> <m|h|d>. example: 1h.",default='30m')
         self.parser.add_option('--env', type='string', help='env, eg: "{env1=xxx, env2=xxx}"')
-        self.parser.add_option('--dis_update', type='string', help='The type is bool, assigned any value representing true', default='true')
         self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default='./')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH)
 
@@ -1935,6 +1972,41 @@ def _do_command(self, obd):
             return self._show_help()
 
 
+class ObdiagGatherAshReportCommand(ObdCommand):
+
+    def __init__(self):
+        super(ObdiagGatherAshReportCommand, self).__init__('ash', 'Gather ash report')
+        self.parser.add_option('--trace_id', type='string',
+                               help="The TRACE.ID of the SQL to be sampled, if left blank or filled with NULL, indicates that TRACE.ID is not restricted.")
+        self.parser.add_option('--sql_id', type='string',
+                               help="The SQL.ID, if left blank or filled with NULL, indicates that SQL.ID is not restricted.")
+        self.parser.add_option('--wait_class', type='string',
+                               help='Event types to be sampled.')
+        self.parser.add_option('--report_type', type='string',
+                               help='Report type, currently only supports text type.', default='TEXT')
+        self.parser.add_option('--from', type='string',
+                               help="specify the start of the time range. format: 'yyyy-mm-dd hh:mm:ss'")
+        self.parser.add_option('--to', type='string',
+                               help="specify the end of the time range. format: 'yyyy-mm-dd hh:mm:ss'")
+        self.parser.add_option('--store_dir', type='string',
+                               help='the dir to store gather result, current dir by default.', default='./')
+        self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir", default=OBDIAG_HOME_PATH)
+
+    def init(self, cmd, args):
+        super(ObdiagGatherAshReportCommand, self).init(cmd, args)
+        self.parser.set_usage('%s <deploy name> [options]' % self.prev_cmd)
+        return self
+
+    @property
+    def lock_mode(self):
+        return LockMode.NO_LOCK
+
+    def _do_command(self, obd):
+        if len(self.cmds) > 0:
+            return obd.obdiag_online_func(self.cmds[0], "gather_ash", self.opts)
+        else:
+            return self._show_help()
+
 class ObdiagAnalyzeMirrorCommand(ObdCommand):
 
     def init(self, cmd, args):
@@ -2013,7 +2085,6 @@ def __init__(self):
         super(ObdiagCheckCommand, self).__init__('check', 'check oceanbase cluster')
         self.parser.add_option('--cases', type='string', help="The name of the check task set that needs to be executed")
         self.parser.add_option('--store_dir', type='string', help='ouput report path', default='./check_report/')
-        self.parser.add_option('--dis_update', type='string', help='The type is bool, assigned any value representing true', default='true')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir", default=OBDIAG_HOME_PATH)
 
     def init(self, cmd, args):
@@ -2048,7 +2119,7 @@ def __init__(self):
         super(ObdiagRcaRunCommand, self).__init__('run', 'to run root cause analysis of oceanbase problem')
         self.parser.add_option('--scene', type='string', help="The name of the rca scene set that needs to be executed")
         self.parser.add_option('--store_dir', type='string', help='ouput result path', default='./rca/')
-        self.parser.add_option('--parameters', type='string', help='parameters')
+        self.parser.add_option('--input_parameters', type='string', help='parameters')
         self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir", default=OBDIAG_HOME_PATH)
 
     def init(self, cmd, args):

_deploy.py

@@ -392,16 +392,17 @@ def _to_cluster_config(cls, component_name, conf, added_servers=None):
     @classmethod
     def merge_config(cls, component_name, config, new_config):
         unmergeable_keys = cls.META_KEYS + ['global', RsyncConfig.RSYNC, ENV]
+        for key in unmergeable_keys:
+            assert key not in new_config, Exception('{} is not allowed to be set in additional conf'.format(key))
         servers = cls._get_servers(config.get('servers', []))
         merge_servers = cls._get_servers(new_config.get('servers', []))
         for server in merge_servers:
             assert server not in servers, Exception('{} is already in cluster'.format(server))
-        new_config['servers'] = config.get('servers', []) + new_config.get('servers', [])
+        config['servers'] = config.get('servers', []) + new_config.get('servers', [])
         merge_server_names = [server.name for server in merge_servers]
-        for key in new_config:
-            assert key not in unmergeable_keys, Exception('{} is not allowed to be set in additional conf'.format(key))
-            assert key in merge_server_names or key == 'servers', Exception('{} is not allowed to be set'.format(key))
-            config[key] = new_config[key]
+        for key in merge_server_names:
+            if key in new_config:
+                config[key] = new_config[key]
         return cls.to_cluster_config(component_name, config, merge_servers)
 
     @classmethod
@@ -656,6 +657,16 @@ def get_depend_config(self, name, server=None, with_default=True):
             config = cluster_config.get_server_conf(server) if server else cluster_config.get_global_conf()
         return deepcopy(config)
 
+    def get_be_depend_config(self, name, server=None, with_default=True):
+        if name not in self._be_depends:
+            return None
+        cluster_config = self._be_depends[name]
+        if with_default:
+            config = cluster_config.get_server_conf_with_default(server) if server else cluster_config.get_global_conf_with_default()
+        else:
+            config = cluster_config.get_server_conf(server) if server else cluster_config.get_global_conf()
+        return deepcopy(config)
+
     def update_server_conf(self, server, key, value, save=True):
         if self._deploy_config is None:
             return False
@@ -1105,7 +1116,11 @@ def get_inner_config(component_name):
     def added_components(self):
         if self._added_components:
             return self._added_components
+        elif self._changed_components or self._removed_components:
+            # It's means that the components have been changed, so must not be added
+            return []
         else:
+            # Maybe the deploy is new, so all components are added
             return self.components.keys()
 
     @property
@@ -1206,13 +1221,16 @@ def scale_out(self, config_path):
                         self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(key))
                         ret = False
                     elif issubclass(type(source_data[key]), dict):
-                        if source_data[key].get('depends', []):
+                        new_depends = source_data[key].get('depends', [])
+                        if new_depends and new_depends != self.components[key].depends:
                             self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(message='depends:{}'.format(key)))
+                            ret = False
                         # temp _depends
                         depends[key] = self.components[key].depends
+
                     if not self._merge_component(key, source_data[key]):
-                            ret = False
-                            continue
+                        ret = False
+
                 for comp in depends:
                     conf = self.components[comp]
                     for name in depends[comp]:
@@ -1472,20 +1490,17 @@ def _merge_component(self, component_name, conf):
             global_attr = deepcopy(src_conf['inner_config'][(ComponentInnerConfig.COMPONENT_GLOBAL_ATTRS)])
         try:
             merged_cluster_config = parser.merge_config(component_name, src_conf['config'], conf)
+            self._src_data[component_name] = src_conf['config']
+            self._set_component(merged_cluster_config)
         except Exception as e:
             self.stdio.exception(err.EC_COMPONENT_FAILED_TO_MERGE_CONFIG.format(message=str(e)))
             return False
-        self.update_component(merged_cluster_config)
+
         cluster_config = self.components[component_name]
         for k in global_attr:
             v = global_attr.get(k)
             cluster_config.update_component_attr(k, v, save=False)
         self._changed_components.append(component_name)
-
-        # 更新depends config
-        for comp in self.components:
-            if component_name in self.components[comp].depends:
-                self.components[comp].add_depend(component_name, merged_cluster_config)
         return True
 
     def change_component_config_style(self, component_name, style):

_environ.py

@@ -56,4 +56,10 @@
 ENV_DISABLE_RSA_ALGORITHMS = 'OBD_DISABLE_RSA_ALGORITHMS'
 
 # set local connection when using host ip. {0/1} 0 - no local connection. 1 - local connection.
-ENV_HOST_IP_MODE = "HOST_IP_MODE"
+ENV_HOST_IP_MODE = "HOST_IP_MODE"
+
+# obdeploy install pre path. default /
+ENV_OBD_INSTALL_PRE = "OBD_INSTALL_PRE"
+
+# obdeploy install path. default /usr/obd/
+ENV_OBD_INSTALL_PATH = "OBD_INSTALL_PATH"