From 1f9a812970631237531779af11bf3f9cab4187dd Mon Sep 17 00:00:00 2001 From: coestre Date: Thu, 18 May 2017 13:41:27 -0500 Subject: [PATCH 1/2] Adding ability to over ride the callsPerMinute in the rate limiter to allow faster git stuff to GHE or non-public githubs --- .gitignore | 2 + .../maven/plugins/core/GitHubProjectMojo.java | 133 ++++++++++-------- .../plugins/core/RateLimitedGitHubClient.java | 11 +- .../plugins/core/ClientCredentialsTest.java | 59 ++++---- .../plugins/core/CustomHostnameTest.java | 13 +- github-site-plugin/pom.xml | 2 +- .../github/maven/plugins/site/SiteMojo.java | 9 +- 7 files changed, 131 insertions(+), 98 deletions(-) diff --git a/.gitignore b/.gitignore index d567ba01..59967f08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +*.iml +.idea bin target diff --git a/github-core/src/main/java/com/github/maven/plugins/core/GitHubProjectMojo.java b/github-core/src/main/java/com/github/maven/plugins/core/GitHubProjectMojo.java index 9785abee..a5b29f10 100644 --- a/github-core/src/main/java/com/github/maven/plugins/core/GitHubProjectMojo.java +++ b/github-core/src/main/java/com/github/maven/plugins/core/GitHubProjectMojo.java @@ -30,8 +30,6 @@ import org.apache.maven.settings.Proxy; import org.apache.maven.settings.Server; import org.apache.maven.settings.Settings; -import org.eclipse.egit.github.core.RepositoryId; -import org.eclipse.egit.github.core.client.GitHubClient; import org.apache.maven.settings.crypto.DefaultSettingsDecryptionRequest; import org.apache.maven.settings.crypto.SettingsDecrypter; import org.apache.maven.settings.crypto.SettingsDecryptionResult; @@ -42,6 +40,9 @@ import org.codehaus.plexus.context.Context; import org.codehaus.plexus.context.ContextException; import org.codehaus.plexus.personality.plexus.lifecycle.phase.Contextualizable; +import org.eclipse.egit.github.core.RepositoryId; +import org.eclipse.egit.github.core.client.GitHubClient; +import org.eclipse.egit.github.core.client.IGitHubConstants; import java.io.IOException; import java.net.InetSocketAddress; @@ -49,7 +50,6 @@ import java.net.URL; import java.text.MessageFormat; import java.util.List; -import org.eclipse.egit.github.core.client.IGitHubConstants; /** * Base GitHub Mojo class to be extended. @@ -134,30 +134,36 @@ protected void info(String message, Throwable throwable) { log.info(message, throwable); } - /** - * Create client - * - * @param host - * @param userName - * @param password - * @param oauth2Token - * @param serverId - * @param settings - * @param session - * @return client - * @throws MojoExecutionException - */ - protected GitHubClient createClient(String host, String userName, - String password, String oauth2Token, String serverId, - Settings settings, MavenSession session) - throws MojoExecutionException { - GitHubClient client; - if (!StringUtils.isEmpty(host)) { - if (isDebug()) - debug("Using custom host: " + host); - client = createClient(host); - } else - client = createClient(); + /** + * Create client + * + * @param host + * @param userName + * @param password + * @param oauth2Token + * @param serverId + * @param settings + * @param session + * @return client + * @throws MojoExecutionException + */ + protected GitHubClient createClient(String host, String userName, + String password, String oauth2Token, String serverId, + Settings settings, MavenSession session) throws MojoExecutionException { + return createClient(host, userName, password, oauth2Token, serverId, settings, session, 20.0); + } + + protected GitHubClient createClient(String host, String userName, + String password, String oauth2Token, String serverId, + Settings settings, MavenSession session, double callsPerMinute) + throws MojoExecutionException { + GitHubClient client; + if (!StringUtils.isEmpty(host)) { + if (isDebug()) + debug("Using custom host: " + host); + client = createClient(host, callsPerMinute); + } else + client = createClient(callsPerMinute); { Proxy proxy = getProxy( settings, serverId, host ); @@ -203,39 +209,48 @@ protected GitHubClient createClient(String host, String userName, "No authentication credentials configured"); } - /** - * Create client - *

- * Subclasses can override to do any custom client configuration - * - * @param hostname - * @return non-null client - * @throws MojoExecutionException - */ - protected GitHubClient createClient(String hostname) - throws MojoExecutionException { - if (!hostname.contains("://")) - return new RateLimitedGitHubClient(hostname); - try { - URL hostUrl = new URL(hostname); - return new RateLimitedGitHubClient(hostUrl.getHost(), hostUrl.getPort(), - hostUrl.getProtocol()); - } catch (MalformedURLException e) { - throw new MojoExecutionException("Could not parse host URL " - + hostname, e); - } - } + /** + * Create client + *

+ * Subclasses can override to do any custom client configuration + * + * @param hostname + * @return non-null client + * @throws MojoExecutionException + */ + protected GitHubClient createClient(String hostname) + throws MojoExecutionException { + return createClient(hostname, 20.0); + } - /** - * Create client - *

- * Subclasses can override to do any custom client configuration - * - * @return non-null client - */ - protected GitHubClient createClient() { - return new RateLimitedGitHubClient(); - } + protected GitHubClient createClient(String hostname, double callsPerMinute) + throws MojoExecutionException { + if (!hostname.contains("://")) + return new RateLimitedGitHubClient(hostname, callsPerMinute); + try { + URL hostUrl = new URL(hostname); + return new RateLimitedGitHubClient(hostUrl.getHost(), hostUrl.getPort(), + hostUrl.getProtocol(), callsPerMinute); + } catch (MalformedURLException e) { + throw new MojoExecutionException("Could not parse host URL " + + hostname, e); + } + } + + /** + * Create client + *

+ * Subclasses can override to do any custom client configuration + * + * @return non-null client + */ + protected GitHubClient createClient() { + return createClient(20.0); + } + + protected GitHubClient createClient(double callsPerMinute) { + return new RateLimitedGitHubClient(callsPerMinute); + } /** * Configure credentials from configured username/password combination diff --git a/github-core/src/main/java/com/github/maven/plugins/core/RateLimitedGitHubClient.java b/github-core/src/main/java/com/github/maven/plugins/core/RateLimitedGitHubClient.java index 5a3ff445..313aed13 100644 --- a/github-core/src/main/java/com/github/maven/plugins/core/RateLimitedGitHubClient.java +++ b/github-core/src/main/java/com/github/maven/plugins/core/RateLimitedGitHubClient.java @@ -12,18 +12,21 @@ public class RateLimitedGitHubClient extends GitHubClientEgit { * AS per https://github.com/octokit/octokit.net/issues/638#issuecomment-67795998, * it seems that GitHub only allow 20 API calls per 1-minute period */ - private RateLimiter rateLimiter = RateLimiter.create(20.0/60.0); + private RateLimiter rateLimiter; - public RateLimitedGitHubClient() { + public RateLimitedGitHubClient(double callsPerMinute) { super(); + rateLimiter = RateLimiter.create(callsPerMinute/60.0); } - public RateLimitedGitHubClient(String hostname) { + public RateLimitedGitHubClient(String hostname, double callsPerMinute) { super(hostname); + rateLimiter = RateLimiter.create(callsPerMinute/60.0); } - public RateLimitedGitHubClient(String hostname, int port, String scheme) { + public RateLimitedGitHubClient(String hostname, int port, String scheme, double callsPerMinute) { super(hostname, port, scheme); + rateLimiter = RateLimiter.create(callsPerMinute/60.0); } @Override diff --git a/github-core/src/test/java/com/github/maven/plugins/core/ClientCredentialsTest.java b/github-core/src/test/java/com/github/maven/plugins/core/ClientCredentialsTest.java index 04f2c574..f98d9d6f 100644 --- a/github-core/src/test/java/com/github/maven/plugins/core/ClientCredentialsTest.java +++ b/github-core/src/test/java/com/github/maven/plugins/core/ClientCredentialsTest.java @@ -56,22 +56,17 @@ private class TestMojo extends GitHubProjectMojo { private final AtomicReference token = new AtomicReference(); - protected GitHubClient createClient() { - try - { - DefaultPlexusContainer container = new DefaultPlexusContainer(); - Context context = container.getContext(); - context.put( PlexusConstants.PLEXUS_KEY, container ); - super.contextualize(context ); - } - catch ( PlexusContainerException pce ) - { - pce.printStackTrace( System.err ); - } - catch ( ContextException ce ) - { - ce.printStackTrace( System.err ); - } + protected GitHubClient createClient(double callsPerMinute) { + try { + DefaultPlexusContainer container = new DefaultPlexusContainer(); + Context context = container.getContext(); + context.put(PlexusConstants.PLEXUS_KEY, container); + super.contextualize(context); + } catch (PlexusContainerException pce) { + pce.printStackTrace(System.err); + } catch (ContextException ce) { + ce.printStackTrace(System.err); + } return new GitHubClient() { public GitHubClient setCredentials(String user, String password) { @@ -85,17 +80,29 @@ public GitHubClient setOAuth2Token(String token) { return super.setOAuth2Token(token); } - }; - } + }; + } - @Override - public GitHubClient createClient(String host, String userName, - String password, String oauth2Token, String serverId, - Settings settings, MavenSession session) - throws MojoExecutionException { - return super.createClient(host, userName, password, oauth2Token, - serverId, settings, session); - } + protected GitHubClient createClient() { + return createClient(20.0); + } + + public GitHubClient createClient(String host, String userName, + String password, String oauth2Token, String serverId, + Settings settings, MavenSession session, double callsPerMinute) + throws MojoExecutionException { + return super.createClient(host, userName, password, oauth2Token, + serverId, settings, session, callsPerMinute); + } + + @Override + public GitHubClient createClient(String host, String userName, + String password, String oauth2Token, String serverId, + Settings settings, MavenSession session) + throws MojoExecutionException { + return this.createClient(host, userName, password, oauth2Token, + serverId, settings, session, 20.0); + } public void execute() throws MojoExecutionException, MojoFailureException { diff --git a/github-core/src/test/java/com/github/maven/plugins/core/CustomHostnameTest.java b/github-core/src/test/java/com/github/maven/plugins/core/CustomHostnameTest.java index f98a153e..49adbdf4 100644 --- a/github-core/src/test/java/com/github/maven/plugins/core/CustomHostnameTest.java +++ b/github-core/src/test/java/com/github/maven/plugins/core/CustomHostnameTest.java @@ -45,23 +45,22 @@ private class TestMojo extends GitHubProjectMojo { private final AtomicReference host = new AtomicReference(); - protected GitHubClient createClient() { + protected GitHubClient createClient(double callsPerMinute) { host.set(null); - return super.createClient(); + return super.createClient(20.0); } - protected GitHubClient createClient(String hostname) - throws MojoExecutionException { + protected GitHubClient createClient(String hostname, double callsPerMinute) throws MojoExecutionException { host.set(hostname); - return super.createClient(hostname); + return super.createClient(hostname, callsPerMinute); } public GitHubClient createClient(String host, String userName, String password, String oauth2Token, String serverId, - Settings settings, MavenSession session) + Settings settings, MavenSession session, double callsPerMinute) throws MojoExecutionException { return super.createClient(host, userName, password, oauth2Token, - serverId, settings, session); + serverId, settings, session, callsPerMinute); } public void execute() throws MojoExecutionException, diff --git a/github-site-plugin/pom.xml b/github-site-plugin/pom.xml index 7a65b3d2..22057d01 100644 --- a/github-site-plugin/pom.xml +++ b/github-site-plugin/pom.xml @@ -14,7 +14,7 @@ Maven plugin that commits files to a branch in a GitHub repository - 0.9 + 0.12 diff --git a/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java b/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java index f09080ca..05f1b2e3 100644 --- a/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java +++ b/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java @@ -243,6 +243,13 @@ public class SiteMojo extends GitHubProjectMojo { */ private boolean skip; + /** + * The number of calls per minute to allow to github. Default to 20/min + * + * @parameter expression="${github.site.callsPerMinute}" + */ + private double callsPerMinute; + /** * Create blob * @@ -328,7 +335,7 @@ public void execute() throws MojoExecutionException { Arrays.toString(paths))); DataService service = new DataService(createClient(host, userName, - password, oauth2Token, server, settings, session)); + password, oauth2Token, server, settings, session, callsPerMinute)); // Write blobs and build tree entries List entries = new ArrayList(paths.length); From f96531063997c6ca64412b13a1d7e199df7ed3d3 Mon Sep 17 00:00:00 2001 From: coestre Date: Thu, 18 May 2017 13:50:46 -0500 Subject: [PATCH 2/2] Adding ability to over ride the callsPerMinute in the rate limiter to allow faster git stuff to GHE or non-public githubs --- README.md | 2 ++ .../src/main/java/com/github/maven/plugins/site/SiteMojo.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 24670817..92648fa3 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ by the settings configuration property in parentheses. * Name of repository * `repositoryOwner` * Owner of repository +* `callsPerMinute` (`github.global.callsPerMinute`) + * Number of calls per minute to allow in the rate limiter. Default is 20, which allows 20 calls/min. Setting this to a value like 1000 will allow 1000 calls/minute to github. Useful if you are using GitHub Enterprise or other self hosted solution where the limit isn't capped at a fixed rate like the public GitHub. *Note:* `repositoryOwner` property and `repositoryName` are optional and will be inferred from the following properties if not specified diff --git a/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java b/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java index 05f1b2e3..c0899d38 100644 --- a/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java +++ b/github-site-plugin/src/main/java/com/github/maven/plugins/site/SiteMojo.java @@ -244,9 +244,10 @@ public class SiteMojo extends GitHubProjectMojo { private boolean skip; /** - * The number of calls per minute to allow to github. Default to 20/min + * The number of calls per minute to allow to github. Default to 20. * * @parameter expression="${github.site.callsPerMinute}" + * default-value="20.0" */ private double callsPerMinute;