feat: add support for chunked uploading (#448)

tested manually with GIF images, but no automated tests in this I hereby declare that this contribution is licensed under Apache License Version 2
redouane59 · Jul 18, 2023 · 05f363e · 05f363e
1 parent f58ec15
commit 05f363e
Show file tree

Hide file tree

Showing 9 changed files with 156 additions and 17 deletions.
diff --git a/pom.xml b/pom.xml
@@ -61,6 +61,7 @@
           <execution>
             <configuration>
               <failOnError>false</failOnError>
+              <additionalOptions>-Xdoclint:none</additionalOptions>
             </configuration>
             <goals>
               <goal>jar</goal>

diff --git a/src/main/java/io/github/redouane59/twitter/ITwitterClientV1.java b/src/main/java/io/github/redouane59/twitter/ITwitterClientV1.java
@@ -12,8 +12,10 @@
 import io.github.redouane59.twitter.dto.tweet.Tweet;
 import io.github.redouane59.twitter.dto.tweet.UploadMediaResponse;
 import java.io.File;
+import java.io.InputStream;
 import java.time.LocalDateTime;
 import java.util.List;
+import java.util.Optional;
 
 public interface ITwitterClientV1 {
 
@@ -113,6 +115,16 @@ public interface ITwitterClientV1 {
    */
   UploadMediaResponse uploadMedia(File media, MediaCategory mediaCategory);
 
+  /**
+   * Upload a media calling https://upload.twitter.com/1.1/media/upload in chunked mode.
+   */
+  Optional<UploadMediaResponse> uploadChunkedMedia(String mediaName, long size, InputStream data, MediaCategory mediaCategory);
+
+  /**
+   * Upload a media calling https://upload.twitter.com/1.1/media/upload in chunked mode.
+   */
+  Optional<UploadMediaResponse> uploadChunkedMedia(File media, MediaCategory mediaCategory);
+
   /**
    * Creates a collection of tweets. See https://api.twitter.com/1.1/collections/create.json
    *
@@ -190,4 +202,4 @@ public interface ITwitterClientV1 {
   @Deprecated
   DmEvent postDm(String text, String userId);
 
-}
+}
diff --git a/src/main/java/io/github/redouane59/twitter/TwitterClient.java b/src/main/java/io/github/redouane59/twitter/TwitterClient.java
@@ -39,21 +39,9 @@
 import io.github.redouane59.twitter.dto.stream.StreamRules;
 import io.github.redouane59.twitter.dto.stream.StreamRules.StreamMeta;
 import io.github.redouane59.twitter.dto.stream.StreamRules.StreamRule;
-import io.github.redouane59.twitter.dto.tweet.HiddenResponse;
+import io.github.redouane59.twitter.dto.tweet.*;
 import io.github.redouane59.twitter.dto.tweet.HiddenResponse.HiddenData;
-import io.github.redouane59.twitter.dto.tweet.LikeResponse;
-import io.github.redouane59.twitter.dto.tweet.MediaCategory;
-import io.github.redouane59.twitter.dto.tweet.RetweetResponse;
-import io.github.redouane59.twitter.dto.tweet.Tweet;
-import io.github.redouane59.twitter.dto.tweet.TweetCountsList;
-import io.github.redouane59.twitter.dto.tweet.TweetList;
 import io.github.redouane59.twitter.dto.tweet.TweetList.TweetMeta;
-import io.github.redouane59.twitter.dto.tweet.TweetParameters;
-import io.github.redouane59.twitter.dto.tweet.TweetSearchResponseV1;
-import io.github.redouane59.twitter.dto.tweet.TweetV1;
-import io.github.redouane59.twitter.dto.tweet.TweetV1Deserializer;
-import io.github.redouane59.twitter.dto.tweet.TweetV2;
-import io.github.redouane59.twitter.dto.tweet.UploadMediaResponse;
 import io.github.redouane59.twitter.dto.user.FollowBody;
 import io.github.redouane59.twitter.dto.user.User;
 import io.github.redouane59.twitter.dto.user.UserActionResponse;
@@ -70,6 +58,8 @@
 import io.github.redouane59.twitter.signature.TwitterCredentials;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -1399,6 +1389,75 @@ public UploadMediaResponse uploadMedia(File imageFile, MediaCategory mediaCatego
     return requestHelperV1.uploadMedia(url, imageFile, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new);
   }
 
+  @Override
+  public Optional<UploadMediaResponse> uploadChunkedMedia(String mediaName, long size, InputStream data, MediaCategory mediaCategory) {
+    try {
+      String type = URLConnection.guessContentTypeFromName(mediaName);
+      String url = urlHelper.getChunkedUploadMediaUrl();
+      Map<String, String> parameters = new HashMap<>();
+      parameters.put("command", "INIT");
+      parameters.put("total_bytes", Long.toString(size));
+      parameters.put("media_type", type);
+      parameters.put("medai_category", mediaCategory.label);
+      UploadMediaResponse initRsp = requestHelperV1.postRequest(url, parameters, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new);
+
+      parameters.clear();
+      parameters.put("command", "APPEND");
+      parameters.put("media_id", initRsp.getMediaId());
+
+      byte[] buf = new byte[(int) Math.min(size, 5 * 1024 * 1024L)]; // 5MB max chunk size
+      int segmentIndex = 0;
+      int count;
+      try {
+        while ((count = data.read(buf)) > 0) {
+          parameters.put("segment_index", Integer.toString(segmentIndex++));
+          requestHelperV1.uploadChunkedMedia(url, parameters, buf, 0, count, Void.class);
+        }
+      } catch (IOException ex) {
+        LOGGER.error("Error occupied on reading media", ex);
+        return Optional.empty();
+      }
+
+      parameters.clear();
+      parameters.put("command", "FINALIZE");
+      parameters.put("media_id", initRsp.getMediaId());
+
+      UploadMediaResponse rsp = requestHelperV1.postRequest(url, parameters, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new);
+      UploadMediaProcessingInfo processing;
+      while ((processing = rsp.getProcessingInfo()) != null && processing.getState().equals("pending")) {
+        try {
+          Thread.sleep(processing.getCheckAfterSecs() * 1000L);
+        } catch (InterruptedException ex) {
+          LOGGER.error("Error occupied on waiting media processing", ex);
+        }
+
+        parameters.clear();
+        parameters.put("command", "STATUS");
+        parameters.put("media_id", initRsp.getMediaId());
+
+        rsp = requestHelperV1.getRequestWithParameters(url, parameters, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new);
+      }
+
+      return Optional.of(rsp);
+    } finally {
+      try {
+        data.close();
+      } catch (IOException ex) {
+        LOGGER.error("Error occupied on closing media stream", ex);
+      }
+    }
+  }
+
+  @Override
+  public Optional<UploadMediaResponse> uploadChunkedMedia(File imageFile, MediaCategory mediaCategory) {
+    try {
+      return uploadChunkedMedia(imageFile.getName(), imageFile.length(), Files.newInputStream(imageFile.toPath()), mediaCategory);
+    } catch (IOException ex) {
+      LOGGER.error("Error occupied on reading media", ex);
+      return Optional.empty();
+    }
+  }
+
   @Override
   public CollectionsResponse collectionsCreate(String name, String description, String collectionUrl, TimeLineOrder timeLineOrder) {
     String              url        = getUrlHelper().getCollectionsCreateUrl();

diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingError.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingError.java
@@ -0,0 +1,17 @@
+package io.github.redouane59.twitter.dto.tweet;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class UploadMediaProcessingError {
+
+  private int    code;
+  private String name;
+  private String message;
+
+}
diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingInfo.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingInfo.java
@@ -0,0 +1,23 @@
+package io.github.redouane59.twitter.dto.tweet;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class UploadMediaProcessingInfo {
+
+  @JsonProperty("state")
+  private String state;
+  @JsonProperty("check_after_secs")
+  private int    checkAfterSecs;
+  @JsonProperty("progress_percent")
+  private int    progressPercent;
+  @JsonProperty("media_key")
+  private String mediaKey;
+  private UploadMediaProcessingError error;
+
+}
diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaResponse.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaResponse.java
@@ -17,5 +17,7 @@ public class UploadMediaResponse {
   private int    size;
   @JsonProperty("media_key")
   private String mediaKey;
+  @JsonProperty("processing_info")
+  private UploadMediaProcessingInfo processingInfo;
 
 }
diff --git a/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java
@@ -131,7 +131,9 @@ public <T> Optional<T> makeRequest(OAuthRequest request, boolean signRequired, C
       } else if (response.getCode() < 200 || response.getCode() > 299) {
         logApiError(request.getVerb().name(), request.getUrl(), stringResponse, response.getCode());
       }
-      result = convert(stringResponse, classType);
+      if (!Void.class.equals(classType)) {
+        result = convert(stringResponse, classType);
+      }
     } catch (IOException ex) {
       LOGGER.error("Error occupied on executing request", ex);
     }
@@ -150,4 +152,4 @@ protected <T> T convert(String json, Class<? extends T> targetClass) throws Json
 
   public abstract <T> Optional<T> getRequestWithParameters(String url, Map<String, String> parameters, Class<T> classType);
 
-}
+}
diff --git a/src/main/java/io/github/redouane59/twitter/helpers/RequestHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/RequestHelper.java
@@ -51,6 +51,18 @@ public <T> Optional<T> uploadMedia(String url, String fileName, byte[] data, Cla
     return makeRequest(request, true, classType);
   }
 
+  public <T> Optional<T> uploadChunkedMedia(String url, Map<String, String> parameters, byte[] media, int off, int len, Class<T> classType) {
+    OAuthRequest request = new OAuthRequest(Verb.POST, url);
+    if (parameters != null) {
+      for (Map.Entry<String, String> param : parameters.entrySet()) {
+        request.addQuerystringParameter(param.getKey(), param.getValue());
+      }
+    }
+    request.initMultipartPayload();
+    request.addBodyPartPayloadInMultipartPayload(new FileByteArrayBodyPartPayload("application/octet-stream", media, off, len, "media"));
+    return makeRequest(request, true, classType);
+  }
+
   public <T> Optional<T> putRequest(String url, String body, Class<T> classType) {
     return makeRequest(Verb.PUT, url, null, body, true, classType);
   }

diff --git a/src/main/java/io/github/redouane59/twitter/helpers/URLHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/URLHelper.java
@@ -3,6 +3,10 @@
 import io.github.redouane59.twitter.dto.tweet.MediaCategory;
 import lombok.Getter;
 
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+
 public class URLHelper {
 
   public static final  int    MAX_LOOKUP                  = 100;
@@ -11,6 +15,7 @@ public class URLHelper {
   public static final  String GET_OAUTH1_TOKEN_URL        = "https://api.twitter.com/oauth/request_token";
   public static final  String GET_OAUTH1_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token";
   private static final String ROOT_URL_V1                 = "https://api.twitter.com/1.1";
+  private static final String UPLOAD_URL_V1               = "https://upload.twitter.com/1.1/media/upload.json";
   public static final  String RATE_LIMIT_URL              = ROOT_URL_V1 + "/application/rate_limit_status.json";
   // v1 legacy
   private static final String IDS_JSON                    = "/ids.json?";
@@ -197,7 +202,13 @@ public String getUserMentionsUrl(String userId) {
   }
 
   public String getUploadMediaUrl(MediaCategory mediaCategory) {
-    return "https://upload.twitter.com/1.1/media/upload.json?media_category=" + mediaCategory.label;
+    return UPLOAD_URL_V1 +
+            "?media_category=" +
+            mediaCategory.label;
+  }
+
+  public String getChunkedUploadMediaUrl() {
+    return UPLOAD_URL_V1;
   }
 
   public String getCollectionsCreateUrl() {