From 2adec23f5215986f73ccc125f0c2f6a25a42371f Mon Sep 17 00:00:00 2001 From: Will Reichert Date: Wed, 13 Oct 2021 11:25:09 -0400 Subject: [PATCH] add suppport for patterns in role hosts --- .../tools/qdup/config/RunConfigBuilder.java | 96 +++++++-- .../qdup/config/RunConfigBuilderTest.java | 187 ++++++++++++++++++ 2 files changed, 271 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/hyperfoil/tools/qdup/config/RunConfigBuilder.java b/src/main/java/io/hyperfoil/tools/qdup/config/RunConfigBuilder.java index 581cd66f..d888ba0a 100644 --- a/src/main/java/io/hyperfoil/tools/qdup/config/RunConfigBuilder.java +++ b/src/main/java/io/hyperfoil/tools/qdup/config/RunConfigBuilder.java @@ -1,6 +1,7 @@ package io.hyperfoil.tools.qdup.config; import io.hyperfoil.tools.qdup.Host; +import io.hyperfoil.tools.qdup.Stage; import io.hyperfoil.tools.qdup.State; import io.hyperfoil.tools.qdup.cmd.Cmd; import io.hyperfoil.tools.qdup.cmd.Script; @@ -117,9 +118,9 @@ public void addError(String error) { errors.add(error); } - public void addErrors(Collection error) { - errors.addAll(error); - } +// public void addErrors(Collection error) { +// errors.addAll(error); +// } public int errorCount() { return errors.size(); @@ -142,6 +143,9 @@ public boolean loadYaml(YamlFile yamlFile) { } }); yamlFile.getRoles().forEach((name, role) -> { + if(role.hasHostExpression()){ + setRoleHostExpession(name,role.getHostExpression().getExpression()); + } role.getHostRefs().forEach(hostRef -> { addHostToRole(name, hostRef); }); @@ -423,7 +427,11 @@ public RunConfig buildConfig(Parser yamlParser) { seenHosts.putIfAbsent(hostShortname, resolvedHost); seenHosts.put(resolvedHost.toString(), resolvedHost);//could duplicate fullyQualified but guarantees to include port } else { - addError("Role " + roleName + " Host " + hostShortname + " was added without a fully qualified host representation matching user@hostName:port\n hosts:" + hostAlias); + if(!Cmd.hasStateReference(hostShortname,null)){ + addError("Role " + roleName + " Host " + hostShortname + " was added without a fully qualified host representation matching user@hostName:port\n hosts:" + hostAlias); + }else{ + + } //WTF, how are we missing a host reference? } } @@ -455,17 +463,47 @@ public RunConfig buildConfig(Parser yamlParser) { i++; } else { //how does an expression end with - - addError("host expresion for " + roleName + " should not end with " + token); + addError("host expression for " + roleName + " should not end with " + token); } } else { - addError("host expressions should be = [+-] ... but " + roleName + " could not parse " + token + " in: " + expession); + if(Cmd.hasStateReference(token,null)){ + String populatedToken = Cmd.populateStateVariables(token,null,getState(),null,new Json()); + if(Cmd.hasStateReference(populatedToken,null)){ + addError("could not fully populate pattern for hosts: "+token); + }else { + if (Json.isJsonLike(populatedToken)) { + Json tokenJson = Json.fromString(populatedToken); + if (tokenJson != null) { + if (tokenJson.isArray()) { + tokenJson.forEach(entry -> { + String str = entry.toString(); + if(hostAliases.containsKey(str)){ + toAdd.add(hostAliases.get(str)); + }else if (Host.parse(str)!=null){ + toAdd.add(Host.parse(str)); + } + }); + } + } + } else if (hostAliases.containsKey(populatedToken)) { + toAdd.add(hostAliases.get(populatedToken)); + } else if (Host.parse(populatedToken) != null) { + toAdd.add(Host.parse(populatedToken)); + } else { + addError("could not identify a host from "+populatedToken); + } + } + }else { + addError("host expressions should be = [+-] ... or a ${{}} pattern but " + roleName + " could not parse " + token + " in: " + expession); + } } } toAdd.removeAll(toRemove); if (!toAdd.isEmpty()) { roleHosts.putAll(roleName, toAdd.stream().map(Host::toString).collect(Collectors.toList())); + toAdd.forEach(host->seenHosts.putIfAbsent(host.toString(),host)); } else { - + addError(roleName+" did not create a host from "+expession); } }); @@ -481,13 +519,45 @@ public RunConfig buildConfig(Parser yamlParser) { roleNames.forEach(roleName -> { roles.putIfAbsent(roleName, new Role(roleName)); roleHosts.get(roleName).forEach(hostRef -> { + if(Cmd.hasStateReference(hostRef,null)){ + hostRef = Cmd.populateStateVariables(hostRef,null,state,null,new Json()); + if(Cmd.hasStateReference(hostRef,null)){ + addError("could not populate "+roleName+" host from "+hostRef); + } + } Host host = seenHosts.get(hostRef); if (host != null) { roles.get(roleName).addHost(host); } else { - //TODO error, missing host definition or assume fully qualified name? + List hostRefs = new ArrayList<>(); + if(Json.isJsonLike(hostRef)){ + Json json = Json.fromString(hostRef); + if(json.isArray()){ + json.values().forEach(v->hostRefs.add(v.toString())); + }else{ + hostRefs.add(hostRef); + } + }else{ + hostRefs.add(hostRef); + } + hostRefs.forEach(ref->{ + if(hostAliases.containsKey(ref)){ + Host newHost = hostAliases.get(ref); + if(newHost!=null){ + seenHosts.putIfAbsent(ref,newHost); + roles.get(roleName).addHost(newHost); + }else{ + addError("could not load "+roleName+" host from alias "+ref); + } + }else if (Host.parse(ref)!=null){ + Host newHost = Host.parse(ref); + seenHosts.putIfAbsent(ref,newHost); + roles.get(roleName).addHost(newHost); + }else { + addError("missing host for " + ref); + } + }); } - }); }); roleSetup.forEach((roleName, cmds) -> { @@ -524,11 +594,13 @@ public RunConfig buildConfig(Parser yamlParser) { summary.addRule("signals",signalCounts); summary.addRule("variables",new UndefinedStateVariables(yamlParser)); summary.addRule("observers",new NonObservingCommands()); - summary.scan(roles.values(),this); - - + if(!errors.isEmpty()){ + errors.forEach(error->{ + summary.addError("", Stage.Pending,"","",error); + }); + } return new RunConfig( getName(), summary.getErrors(), diff --git a/src/test/java/io/hyperfoil/tools/qdup/config/RunConfigBuilderTest.java b/src/test/java/io/hyperfoil/tools/qdup/config/RunConfigBuilderTest.java index 1fa1af08..a4952541 100644 --- a/src/test/java/io/hyperfoil/tools/qdup/config/RunConfigBuilderTest.java +++ b/src/test/java/io/hyperfoil/tools/qdup/config/RunConfigBuilderTest.java @@ -25,6 +25,193 @@ public class RunConfigBuilderTest extends SshTestBase { + @Test + public void role_hosts_pattern_as_expression_existing_host(){ + Parser parser = Parser.getInstance(); + RunConfigBuilder builder = getBuilder(); + builder.loadYaml(parser.loadFile("first",stream(""+ + "scripts:", + " foo:", + " - sh: pwd", + "hosts:", + " foo: me@localhost", + "roles:", + " doit:", + " hosts: ${{hostname}}", + " setup-scripts:", + " - foo", + "states:", + " hostname: foo" + ))); + RunConfig config = builder.buildConfig(parser); + assertFalse("unexpected errors:\n"+config.getErrors().stream().map(Objects::toString).collect(Collectors.joining("\n")),config.hasErrors()); + assertEquals("expect 2 roles because of ALL",2,config.getRoles().size()); + Role role = config.getRole("doit"); + assertNotNull("doit it should be a role: "+config.getRoleNames(),role); + List declaredHosts = role.getDeclaredHosts(); + assertEquals("expect 1 host for role",1,declaredHosts.size()); + Host first = declaredHosts.get(0); + assertNotNull("host should not be null",first); + assertEquals("me",first.getUserName()); + assertEquals("localhost",first.getHostName()); + } + @Test + public void role_hosts_pattern_as_expression_new_host(){ + Parser parser = Parser.getInstance(); + RunConfigBuilder builder = getBuilder(); + builder.loadYaml(parser.loadFile("first",stream(""+ + "scripts:", + " foo:", + " - sh: pwd", + "hosts:", + " foo: you@localhost", + "roles:", + " doit:", + " hosts: ${{hostname}}", + " setup-scripts:", + " - foo", + "states:", + " hostname: me@localhost" + ))); + RunConfig config = builder.buildConfig(parser); + assertFalse("unexpected errors:\n"+config.getErrors().stream().map(Objects::toString).collect(Collectors.joining("\n")),config.hasErrors()); + assertEquals("expect 2 roles because of ALL",2,config.getRoles().size()); + Role role = config.getRole("doit"); + assertNotNull("doit it should be a role: "+config.getRoleNames(),role); + List declaredHosts = role.getDeclaredHosts(); + assertEquals("expect 1 host for role",1,declaredHosts.size()); + Host first = declaredHosts.get(0); + assertNotNull("host should not be null",first); + assertEquals("me",first.getUserName()); + assertEquals("localhost",first.getHostName()); + } + @Test + public void role_hosts_pattern_as_expression_new_host_array(){ + Parser parser = Parser.getInstance(); + RunConfigBuilder builder = getBuilder(); + builder.loadYaml(parser.loadFile("first",stream(""+ + "scripts:", + " foo:", + " - sh: pwd", + "hosts:", + " foo: you@localhost", + "roles:", + " doit:", + " hosts: ${{hostname}}", + " setup-scripts:", + " - foo", + "states:", + " hostname: ['me@localhost','he@localhost','she@localhost']" + ))); + RunConfig config = builder.buildConfig(parser); + assertFalse("unexpected errors:\n"+config.getErrors().stream().map(Objects::toString).collect(Collectors.joining("\n")),config.hasErrors()); + assertEquals("expect 2 roles because of ALL",2,config.getRoles().size()); + Role role = config.getRole("doit"); + assertNotNull("doit it should be a role: "+config.getRoleNames(),role); + List declaredHosts = role.getDeclaredHosts(); + assertEquals("expect 3 host for role",3,declaredHosts.size()); + assertTrue("hosts should container me@localhost: "+declaredHosts,declaredHosts.contains(Host.parse("me@localhost"))); + assertTrue("hosts should container he@localhost: "+declaredHosts,declaredHosts.contains(Host.parse("he@localhost"))); + assertTrue("hosts should container she@localhost: "+declaredHosts,declaredHosts.contains(Host.parse("she@localhost"))); + } + + @Test + public void role_hosts_pattern_in_list_existing_host(){ + Parser parser = Parser.getInstance(); + RunConfigBuilder builder = getBuilder(); + builder.loadYaml(parser.loadFile("first",stream(""+ + "scripts:", + " foo:", + " - sh: pwd", + "hosts:", + " foo: me@localhost", + "roles:", + " doit:", + " hosts:", + " - ${{hostname}}", + " setup-scripts:", + " - foo", + "states:", + " hostname: foo" + ))); + RunConfig config = builder.buildConfig(parser); + assertFalse("unexpected errors:\n"+config.getErrors().stream().map(Objects::toString).collect(Collectors.joining("\n")),config.hasErrors()); + assertEquals("expect 2 roles because of ALL",2,config.getRoles().size()); + Role role = config.getRole("doit"); + assertNotNull("doit it should be a role: "+config.getRoleNames(),role); + List declaredHosts = role.getDeclaredHosts(); + assertEquals("expect 1 host for role",1,declaredHosts.size()); + Host first = declaredHosts.get(0); + assertNotNull("host should not be null",first); + assertEquals("me",first.getUserName()); + assertEquals("localhost",first.getHostName()); + } + + @Test + public void role_hosts_pattern_in_list_new_host(){ + Parser parser = Parser.getInstance(); + RunConfigBuilder builder = getBuilder(); + builder.loadYaml(parser.loadFile("first",stream(""+ + "scripts:", + " foo:", + " - sh: pwd", + "hosts:", + " foo: you@localhost", + "roles:", + " doit:", + " hosts:", + " - ${{hostname}}", + " setup-scripts:", + " - foo", + "states:", + " hostname: me@localhost" + ))); + RunConfig config = builder.buildConfig(parser); + assertFalse("unexpected errors:\n"+config.getErrors().stream().map(Objects::toString).collect(Collectors.joining("\n")),config.hasErrors()); + assertEquals("expect 2 roles because of ALL",2,config.getRoles().size()); + Role role = config.getRole("doit"); + assertNotNull("doit it should be a role: "+config.getRoleNames(),role); + List declaredHosts = role.getDeclaredHosts(); + assertEquals("expect 1 host for role",1,declaredHosts.size()); + Host first = declaredHosts.get(0); + assertNotNull("host should not be null",first); + assertEquals("me",first.getUserName()); + assertEquals("localhost",first.getHostName()); + } + + @Test + public void role_hosts_pattern_in_list_new_host_array(){ + Parser parser = Parser.getInstance(); + RunConfigBuilder builder = getBuilder(); + builder.loadYaml(parser.loadFile("first",stream(""+ + "scripts:", + " foo:", + " - sh: pwd", + "hosts:", + " foo: you@localhost", + "roles:", + " doit:", + " hosts:", + " - ${{hostname}}", + " setup-scripts:", + " - foo", + "states:", + " hostname: ['me@localhost','he@localhost','she@localhost']" + ))); + RunConfig config = builder.buildConfig(parser); + assertFalse("unexpected errors:\n"+config.getErrors().stream().map(Objects::toString).collect(Collectors.joining("\n")),config.hasErrors()); + assertEquals("expect 2 roles because of ALL",2,config.getRoles().size()); + Role role = config.getRole("doit"); + assertNotNull("doit it should be a role: "+config.getRoleNames(),role); + List declaredHosts = role.getDeclaredHosts(); + assertEquals("expect 3 host for role",3,declaredHosts.size()); + assertTrue("hosts should container me@localhost: "+declaredHosts,declaredHosts.contains(Host.parse("me@localhost"))); + assertTrue("hosts should container he@localhost: "+declaredHosts,declaredHosts.contains(Host.parse("he@localhost"))); + assertTrue("hosts should container she@localhost: "+declaredHosts,declaredHosts.contains(Host.parse("she@localhost"))); + + } + + @Test public void multiple_yaml_override_state(){ Parser parser = Parser.getInstance();