diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b42d3beb8..e8cc30d9b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,16 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.42.2] - 2023-05-11 +- Fix synchronization on `RequestContext` to prevent `ConcurrentModificationException`. + +## [29.42.1] - 2023-05-11 +- Add support for returning location of schema elements from the PDL schema encoder. + +## [29.42.0] - 2023-05-02 +- Remove the overriding of content-length for HEADER requests as per HTTP Spec + More details about this issue can be found @ https://jira01.corp.linkedin.com:8443/browse/SI-31814 + ## [29.41.12] - 2023-04-06 - Introduce `@extension.injectedUrnParts` ER annotation. - This will be used as the replacement for using `@extension.params` to specify injected URN parts. @@ -5458,7 +5468,10 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.41.12...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.42.2...master +[29.42.2]: https://github.com/linkedin/rest.li/compare/v29.42.1...v29.42.2 +[29.42.1]: https://github.com/linkedin/rest.li/compare/v29.42.0...v29.42.1 +[29.42.0]: https://github.com/linkedin/rest.li/compare/v29.41.12...v29.42.0 [29.41.12]: https://github.com/linkedin/rest.li/compare/v29.41.11...v29.41.12 [29.41.11]: https://github.com/linkedin/rest.li/compare/v29.41.10...v29.41.11 [29.41.10]: https://github.com/linkedin/rest.li/compare/v29.41.9...v29.41.10 diff --git a/build.gradle b/build.gradle index 3fab2df6f5..cfe88a689a 100644 --- a/build.gradle +++ b/build.gradle @@ -144,7 +144,7 @@ allprojects { throw new GradleScriptException("Pegasus required Java 8 or later to build, current version: ${JavaVersion.current()}", null) } // for all supported versions that we test build, fail the build if any compilation warnings are reported - compile.options.compilerArgs = ['-Xlint', '-Xlint:-path', '-Xlint:-static', '-Werror'] + compile.options.compilerArgs = ['-Xlint', '-Xlint:-path', '-Xlint:-static'] } tasks.withType(Javadoc) diff --git a/build_script/restModel.gradle b/build_script/restModel.gradle index d4a23865fa..9b374408cb 100644 --- a/build_script/restModel.gradle +++ b/build_script/restModel.gradle @@ -79,7 +79,7 @@ project.sourceSets.all { SourceSet sourceSet -> project.tasks[sourceSet.compileJavaTaskName].dependsOn(rootProject.ext.build.restModelGenerateTasks[sourceSet]) } - final Task jarTask = project.tasks[sourceSet.getTaskName('', 'jar')] + final Task jarTask = project.tasks[sourceSet.getName().endsWith('11') ? 'jar' : sourceSet.getTaskName('', 'jar')] jarTask.from(inputParentDirPath) { include "${pegasusDirName}${File.separatorChar}**${File.separatorChar}*.pdsc" include "${pegasusDirName}${File.separatorChar}**${File.separatorChar}*.pdl" diff --git a/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java b/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java index a7f0bca7ad..7d2c04f8c7 100644 --- a/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java +++ b/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java @@ -18,11 +18,14 @@ import com.linkedin.data.DataList; import com.linkedin.data.DataMap; +import com.linkedin.data.schema.grammar.PdlSchemaParser; +import com.linkedin.util.LineColumnNumberWriter; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.util.Collections; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -107,15 +110,45 @@ PdlBuilder newBuilderInstance(Writer writer) private String _namespace = ""; private String _package = ""; + private final boolean _trackWriteLocations; + + private final Map _writeLocations; + /** * Construct a .pdl source code encoder. + * The encoding style defaults to {@link EncodingStyle#INDENTED} but may be changed by calling + * {@link #setEncodingStyle(EncodingStyle)}. * * @param out provides the encoded .pdl destination. */ public SchemaToPdlEncoder(Writer out) { - _writer = out; - _encodingStyle = EncodingStyle.INDENTED; + this(out, false); + } + + /** + * Construct a .pdl source code encoder with the option to track line/column of schema elements during writing. + * The encoding style defaults to {@link EncodingStyle#INDENTED} but may be changed by calling + * {@link #setEncodingStyle(EncodingStyle)}. + * + * @param out provides the encoded .pdl destination. + * @param returnContextLocations Enable recording the context locations of schema elements during parsing. The + * locations can be retrieved using {@link #getWriteLocations()} after parsing. + */ + public SchemaToPdlEncoder(Writer out, boolean returnContextLocations) + { + if (returnContextLocations) + { + _writeLocations = new IdentityHashMap<>(); + // Wrap the Writer to track line/column numbers to report to elementWriteListener + _writer = new LineColumnNumberWriter(out); + } else + { + _writer = out; + _writeLocations = Collections.emptyMap(); + } + setEncodingStyle(EncodingStyle.INDENTED); + _trackWriteLocations = returnContextLocations; } /** @@ -126,6 +159,18 @@ public SchemaToPdlEncoder(Writer out) public void setEncodingStyle(EncodingStyle encodingStyle) { _encodingStyle = encodingStyle; + + // When counting column numbers, CompactPDLBuilder treats ',' as whitespace + if (_writer instanceof LineColumnNumberWriter) + { + if (_encodingStyle == EncodingStyle.COMPACT) + { + ((LineColumnNumberWriter) _writer).setIsWhitespaceFunction(c -> Character.isWhitespace(c) || c == ','); + } else + { + ((LineColumnNumberWriter) _writer).setIsWhitespaceFunction(Character::isWhitespace); + } + } } /** @@ -150,10 +195,12 @@ public void encode(DataSchema schema) throws IOException { if (hasNamespace) { + markSchemaElementStartLocation(); _builder.write("namespace") .writeSpace() .writeIdentifier(namedSchema.getNamespace()) .newline(); + recordSchemaElementLocation(namedSchema.getNamespace()); _namespace = namedSchema.getNamespace(); } if (hasPackage) @@ -220,12 +267,14 @@ private void writeInlineSchema(DataSchema schema) throws IOException .increaseIndent(); if (hasNamespaceOverride) { + markSchemaElementStartLocation(); _builder .indent() .write("namespace") .writeSpace() .writeIdentifier(namedSchema.getNamespace()) .newline(); + recordSchemaElementLocation(namedSchema.getNamespace()); _namespace = namedSchema.getNamespace(); } if (hasPackageOverride) @@ -291,8 +340,14 @@ private void writeInlineSchema(DataSchema schema) throws IOException } } + public Map getWriteLocations() + { + return _writeLocations; + } + private void writeRecord(RecordDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("record") .writeSpace() @@ -327,6 +382,7 @@ private void writeRecord(RecordDataSchema schema) throws IOException { writeIncludes(schema, includes); } + recordSchemaElementLocation(schema); } /** @@ -335,6 +391,7 @@ private void writeRecord(RecordDataSchema schema) throws IOException */ private void writeField(RecordDataSchema.Field field) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(field); _builder.indent() .writeIdentifier(field.getName()) @@ -353,6 +410,7 @@ private void writeField(RecordDataSchema.Field field) throws IOException .writeSpace() .writeJson(field.getDefault(), field.getType()); } + recordSchemaElementLocation(field); _builder.newline(); } @@ -382,6 +440,7 @@ private void writeEnum(EnumDataSchema schema) throws IOException DataSchemaConstants.DEPRECATED_SYMBOLS_KEY, properties.get(DataSchemaConstants.DEPRECATED_SYMBOLS_KEY)); + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("enum") .writeSpace() @@ -395,6 +454,7 @@ private void writeEnum(EnumDataSchema schema) throws IOException for (String symbol : schema.getSymbols()) { + markSchemaElementStartLocation(); String docString = docs.get(symbol); DataMap symbolProperties = coercePropertyToDataMapOrFail(schema, DataSchemaConstants.SYMBOL_PROPERTIES_KEY + "." + symbol, @@ -414,24 +474,29 @@ private void writeEnum(EnumDataSchema schema) throws IOException _builder.indent() .writeIdentifier(symbol) .newline(); + recordSchemaElementLocation(symbol); } _builder.decreaseIndent() .indent() .write("}"); + recordSchemaElementLocation(schema); } private void writeFixed(FixedDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("fixed") .writeSpace() .writeIdentifier(schema.getName()) .writeSpace() .write(String.valueOf(schema.getSize())); + recordSchemaElementLocation(schema); } private void writeTyperef(TyperefDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("typeref") .writeSpace() @@ -441,24 +506,29 @@ private void writeTyperef(TyperefDataSchema schema) throws IOException .writeSpace(); DataSchema ref = schema.getRef(); writeReferenceOrInline(ref, schema.isRefDeclaredInline()); + recordSchemaElementLocation(schema); } private void writeMap(MapDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeProperties(schema.getProperties()); _builder.write("map[string") .writeComma() .writeSpace(); writeReferenceOrInline(schema.getValues(), schema.isValuesDeclaredInline()); _builder.write("]"); + recordSchemaElementLocation(schema); } private void writeArray(ArrayDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeProperties(schema.getProperties()); _builder.write("array["); writeReferenceOrInline(schema.getItems(), schema.isItemsDeclaredInline()); _builder.write("]"); + recordSchemaElementLocation(schema); } /** @@ -467,6 +537,7 @@ private void writeArray(ArrayDataSchema schema) throws IOException */ private void writeUnion(UnionDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeProperties(schema.getProperties()); _builder.write("union["); final boolean useMultilineFormat = schema.areMembersAliased() || schema.getMembers().size() >= UNION_MULTILINE_THRESHOLD; @@ -496,6 +567,7 @@ private void writeUnion(UnionDataSchema schema) throws IOException .indent(); } _builder.write("]"); + recordSchemaElementLocation(schema); } /** @@ -505,6 +577,7 @@ private void writeUnion(UnionDataSchema schema) throws IOException */ private void writeUnionMember(UnionDataSchema.Member member, boolean useMultilineFormat) throws IOException { + markSchemaElementStartLocation(); if (member.hasAlias()) { if (StringUtils.isNotBlank(member.getDoc()) || !member.getProperties().isEmpty() || member.isDeclaredInline()) @@ -524,6 +597,7 @@ else if (useMultilineFormat) _builder.indent(); } writeReferenceOrInline(member.getType(), member.isDeclaredInline()); + recordSchemaElementLocation(member); } private void writePrimitive(PrimitiveDataSchema schema) throws IOException @@ -865,4 +939,25 @@ else if (_namespace.equals(schema.getNamespace()) && !_importsByLocalName.contai _builder.writeIdentifier(schema.getFullName()); } } + + void markSchemaElementStartLocation() + { + if (_trackWriteLocations) + { + ((LineColumnNumberWriter) _writer).saveCurrentPosition(); + } + } + + private void recordSchemaElementLocation(Object schemaElement) + { + if (_trackWriteLocations) + { + LineColumnNumberWriter.CharacterPosition startPosition = ((LineColumnNumberWriter) _writer).popSavedPosition(); + LineColumnNumberWriter.CharacterPosition endPosition = + ((LineColumnNumberWriter) _writer).getLastNonWhitespacePosition(); + _writeLocations.put(schemaElement, + new PdlSchemaParser.ParseLocation(startPosition.getLine(), startPosition.getColumn(), endPosition.getLine(), + endPosition.getColumn())); + } + } } diff --git a/data/src/main/java/com/linkedin/util/LineColumnNumberWriter.java b/data/src/main/java/com/linkedin/util/LineColumnNumberWriter.java new file mode 100644 index 0000000000..1e29d7b21f --- /dev/null +++ b/data/src/main/java/com/linkedin/util/LineColumnNumberWriter.java @@ -0,0 +1,226 @@ +package com.linkedin.util; + +import java.io.IOException; +import java.io.Writer; +import java.util.Objects; +import java.util.Stack; +import java.util.function.Predicate; + + +/** + * Wraps a {@link Writer} and tracks current line and column numbers + */ +public final class LineColumnNumberWriter extends Writer +{ + + private final Writer _writer; + private final Stack _savedPositionStack = new Stack<>(); + private int _column; + private int _line; + private int _previousChar; + private Predicate _isWhitespaceFunction; + private final CharacterPosition _lastNonWhitespacePosition; + + /** + * Creates a new writer. + * + * @param out a Writer object to provide the underlying stream. + */ + public LineColumnNumberWriter(Writer out) + { + _writer = out; + _column = 1; + _line = 1; + _previousChar = -1; + _isWhitespaceFunction = (Character::isWhitespace); + _lastNonWhitespacePosition = new CharacterPosition(0, 0); + } + + /** + * Returns 1 based indices of row and column next character will be written to + */ + public CharacterPosition getCurrentPosition() + { + return new CharacterPosition(_line, _column); + } + + /** + * Returns 1 based indices of last row and column ignoring trailing whitespace characters + */ + public CharacterPosition getLastNonWhitespacePosition() + { + return _lastNonWhitespacePosition; + } + + /** + * Saves current row and column to be retrieved later by calling {@link #popSavedPosition()} + * + * Saved positions are stored in a stack so that calls to saveCurrentPosition() and + * {@link #popSavedPosition()} can be nested. Saved positions are adjusted to skip whitespace to make it + * easier to get actual token start positions in indented output. If you call saveCurrentPosition() at column x + * and then write four spaces followed by non-whitespace, the column number returned by + * {@link #popSavedPosition()} will be x + 4. + */ + public void saveCurrentPosition() + { + _savedPositionStack.push(new CharacterPosition(_line, _column)); + } + + /** + * Retrieves row and column from the last time {@link #saveCurrentPosition()} was called + */ + public CharacterPosition popSavedPosition() + { + return _savedPositionStack.pop(); + } + + /** + * Override definition of whitespace used to adjust character positions to skip + * whitespace. By default, the definition of whitespace is provided by {@link java.lang.Character#isWhitespace} + */ + public void setIsWhitespaceFunction(Predicate isWhitespaceFunction) + { + _isWhitespaceFunction = isWhitespaceFunction; + } + + @Override + public void write(char[] cbuf, int off, int len) throws IOException + { + _writer.write(cbuf, off, len); + for (; len > 0; len--) + { + char c = cbuf[off++]; + int lastLine = _line; + int lastColumn = _column; + updateCurrentPosition(c); + _previousChar = c; + if (_isWhitespaceFunction.test(c)) + { + updateSavedPositionsForWhitespace(lastLine, lastColumn); + } else + { + _lastNonWhitespacePosition.line = lastLine; + _lastNonWhitespacePosition.column = lastColumn; + } + } + } + + @Override + public void flush() throws IOException + { + _writer.flush(); + } + + @Override + public void close() throws IOException + { + _writer.close(); + } + + @Override + public String toString() + { + return _writer.toString(); + } + + private void updateCurrentPosition(char c) + { + if (_previousChar == '\r') + { + if (c == '\n') + { + _column = 1; + } else + { + _column = 2; + } + } else if (c == '\n' || c == '\r') + { + _column = 1; + ++_line; + } else + { + ++_column; + } + } + + /** + * Any saved positions that are equal to the current row and column are set to the current position in order to + * remove leading whitespace. Once the first non-whitespace character is written, the current position will be + * different from any saved positions and the current position will advance. + */ + private void updateSavedPositionsForWhitespace(int lastLine, int lastColumn) + { + for (int i = _savedPositionStack.size() - 1; i >= 0; --i) + { + CharacterPosition savedCharacterPosition = _savedPositionStack.get(i); + if (savedCharacterPosition.line == lastLine && savedCharacterPosition.column == lastColumn) + { + savedCharacterPosition.line = _line; + savedCharacterPosition.column = _column; + } else + { + break; + } + } + } + + /** + * Row and column numbers of a character in Writer output + */ + public static class CharacterPosition + { + + private int line; + private int column; + + CharacterPosition(int line, int column) + { + this.line = line; + this.column = column; + } + + /** + * 1-based index of line in writer output + */ + public int getLine() + { + return line; + } + + /** + * 1-based index of column in writer output + */ + public int getColumn() + { + return column; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + CharacterPosition characterPosition = (CharacterPosition) o; + return line == characterPosition.line && column == characterPosition.column; + } + + @Override + public int hashCode() + { + return Objects.hash(line, column); + } + + @Override + public String toString() + { + return "CharacterPosition{" + "line=" + line + ", column=" + column + '}'; + } + } +} diff --git a/data/src/test/java/com/linkedin/util/TestLineColumnNumberWriter.java b/data/src/test/java/com/linkedin/util/TestLineColumnNumberWriter.java new file mode 100644 index 0000000000..c3010149cc --- /dev/null +++ b/data/src/test/java/com/linkedin/util/TestLineColumnNumberWriter.java @@ -0,0 +1,51 @@ +package com.linkedin.util; + +import java.io.IOException; +import java.io.StringWriter; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class TestLineColumnNumberWriter +{ + + @Test + public void testHandlesDifferentNewlines() throws IOException + { + LineColumnNumberWriter writer = new LineColumnNumberWriter(new StringWriter()); + writer.write("1\n2\n3\n"); + Assert.assertEquals(writer.getCurrentPosition(), new LineColumnNumberWriter.CharacterPosition(4, 1)); + writer.write("1\r\n2\r\n3\r\n"); + Assert.assertEquals(writer.getCurrentPosition(), new LineColumnNumberWriter.CharacterPosition(7, 1)); + writer.write("1\r2\r3\r"); + Assert.assertEquals(writer.getCurrentPosition(), new LineColumnNumberWriter.CharacterPosition(10, 1)); + } + + @Test + public void testSavedPositionIgnoresLeadingWhitespace() throws IOException + { + LineColumnNumberWriter writer = new LineColumnNumberWriter(new StringWriter()); + writer.write("123\n"); + writer.saveCurrentPosition(); + writer.saveCurrentPosition(); + writer.write(" \n "); + writer.write("456"); + writer.saveCurrentPosition(); + writer.write(" 789"); + Assert.assertEquals(writer.popSavedPosition(), new LineColumnNumberWriter.CharacterPosition(3, 8)); + Assert.assertEquals(writer.popSavedPosition(), new LineColumnNumberWriter.CharacterPosition(3, 2)); + Assert.assertEquals(writer.popSavedPosition(), new LineColumnNumberWriter.CharacterPosition(3, 2)); + } + + @Test + public void testGetLastNonWhitespacePosition() throws IOException + { + LineColumnNumberWriter writer = new LineColumnNumberWriter(new StringWriter()); + writer.write("123"); + Assert.assertEquals(writer.getLastNonWhitespacePosition(), new LineColumnNumberWriter.CharacterPosition(1, 3)); + writer.write("\n "); + Assert.assertEquals(writer.getLastNonWhitespacePosition(), new LineColumnNumberWriter.CharacterPosition(1, 3)); + writer.write("4"); + Assert.assertEquals(writer.getLastNonWhitespacePosition(), new LineColumnNumberWriter.CharacterPosition(2, 2)); + } +} diff --git a/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java b/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java index 7d12f35666..ae22fdef75 100644 --- a/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java +++ b/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java @@ -21,7 +21,10 @@ import com.linkedin.data.schema.AbstractSchemaParser; import com.linkedin.data.schema.DataSchema; import com.linkedin.data.schema.DataSchemaResolver; +import com.linkedin.data.schema.NamedDataSchema; +import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.schema.SchemaToPdlEncoder; +import com.linkedin.data.schema.UnionDataSchema; import com.linkedin.data.schema.grammar.PdlSchemaParser; import com.linkedin.data.schema.resolver.MultiFormatDataSchemaResolver; import com.linkedin.pegasus.generator.test.idl.EncodingStyle; @@ -31,6 +34,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.StringWriter; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -149,6 +153,75 @@ private void assertRoundTrip(String relativeName) throws IOException } } + @Test(dataProvider = "pdlFilePaths") + public void testTrackWriteLocations(String pdlFilePath) throws IOException + { + assertRoundTripLineColumnNumbersMatch(pdlFilePath); + } + + private void assertRoundTripLineColumnNumbersMatch(String relativeName) throws IOException + { + String fullName = "com.linkedin.pegasus.generator.test.idl." + relativeName; + File file = new File(pegasusSrcDir, "/" + fullName.replace('.', '/') + ".pdl"); + + TypeReferenceFormat referenceFormat = TypeReferenceFormat.PRESERVE; + + // Test all encoding styles + for (SchemaToPdlEncoder.EncodingStyle encodingStyle : SchemaToPdlEncoder.EncodingStyle.values()) + { + String encoded = readAndStandardizeFormat(file, referenceFormat, encodingStyle); + + DataSchemaResolver resolver = MultiFormatDataSchemaResolver.withBuiltinFormats(pegasusSrcDir.getAbsolutePath()); + PdlSchemaParser parser = new PdlSchemaParser(resolver, true); + parser.parse(encoded); + Map parsedLocations = parser.getParseLocations(); + DataSchema parsed = extractSchema(parser, file.getAbsolutePath()); + + StringWriter writer = new StringWriter(); + SchemaToPdlEncoder encoder = new SchemaToPdlEncoder(writer, true); + encoder.setTypeReferenceFormat(referenceFormat); + encoder.setEncodingStyle(encodingStyle); + encoder.encode(parsed); + Map writeLocations = encoder.getWriteLocations(); + + for (Map.Entry expected : parsedLocations.entrySet()) + { + PdlSchemaParser.ParseLocation actual = writeLocations.get(expected.getKey()); + + Assert.assertNotNull(actual, + "Missing location for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getStartLine(), expected.getValue().getStartLine(), + "Start line for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getStartColumn(), expected.getValue().getStartColumn(), + "Start col for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getEndLine(), expected.getValue().getEndLine(), + "End line for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getEndColumn(), expected.getValue().getEndColumn(), + "End col for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + } + + Assert.assertEquals(parsedLocations.size(), writeLocations.size(), + "Different numer of element locations for " + file.getAbsolutePath()); + } + } + + private String readAndStandardizeFormat(File file, TypeReferenceFormat typeReferenceFormat, + SchemaToPdlEncoder.EncodingStyle encodingStyle) throws IOException + { + DataSchema parsed = parseSchema(file); + StringWriter writer = new StringWriter(); + SchemaToPdlEncoder encoder = new SchemaToPdlEncoder(writer); + encoder.setEncodingStyle(encodingStyle); + encoder.setTypeReferenceFormat(typeReferenceFormat); + encoder.encode(parsed); + return writer.toString(); + } + private DataSchema parseSchema(File file) throws IOException { DataSchemaResolver resolver = MultiFormatDataSchemaResolver.withBuiltinFormats(pegasusSrcDir.getAbsolutePath()); diff --git a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy index dad8526c8d..2c4926798e 100644 --- a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy +++ b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy @@ -96,6 +96,6 @@ class PegasusPluginCacheabilityTest extends Specification { preparedSchema.exists() where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2' ] } } diff --git a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy index ea6002f4ea..bbd1dca7f3 100644 --- a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy +++ b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy @@ -34,7 +34,7 @@ class PegasusPluginIntegrationTest extends Specification { result.task(':mainDataTemplateJar').outcome == SUCCESS where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] } @Unroll @@ -102,7 +102,7 @@ class PegasusPluginIntegrationTest extends Specification { assertZipContains(dataTemplateArtifact, 'extensions/com/linkedin/LatLongExtensions.pdl') where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] } def 'mainCopySchema task will remove stale PDSC'() { @@ -262,7 +262,7 @@ class PegasusPluginIntegrationTest extends Specification { result.task(':impl:compileJava').outcome == SUCCESS where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] } private static boolean assertZipContains(File zip, String path) { diff --git a/gradle.properties b/gradle.properties index 9b6ec08860..8f88ec9791 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.41.12 +version=29.42.2 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true diff --git a/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java b/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java index 707622e383..717f427352 100644 --- a/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java +++ b/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java @@ -54,12 +54,14 @@ public RequestContext() */ public RequestContext(RequestContext other) { - _localAttrs = Collections.synchronizedMap(new HashMap<>(other._localAttrs)); + synchronized (other._localAttrs) { + _localAttrs = Collections.synchronizedMap(new HashMap<>(other._localAttrs)); + } } private RequestContext(Map localAttrs) { - _localAttrs = localAttrs; + _localAttrs = Collections.synchronizedMap(localAttrs); } /** @@ -117,7 +119,7 @@ public RequestContext clone() public boolean equals(Object o) { return (o instanceof RequestContext) && - ((RequestContext)o)._localAttrs.equals(this._localAttrs); + ((RequestContext)o)._localAttrs.equals(this._localAttrs); } @Override diff --git a/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java b/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java index 2b5396962f..ebab27f33e 100644 --- a/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java +++ b/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java @@ -54,6 +54,7 @@ */ public abstract class AbstractClient implements Client { + public static final String HTTP_HEAD_METHOD = "HEAD"; @Override public Future restRequest(RestRequest request) @@ -88,8 +89,10 @@ public void restRequest(RestRequest request, RequestContext requestContext, Call // IS_FULL_REQUEST flag, if set true, would result in the request being sent without using chunked transfer encoding // This is needed as the legacy R2 server (before 2.8.0) does not support chunked transfer encoding. requestContext.putLocalAttr(R2Constants.IS_FULL_REQUEST, true); + + boolean addContentLengthHeader = !HTTP_HEAD_METHOD.equalsIgnoreCase(request.getMethod()); // here we add back the content-length header for the response because some client code depends on this header - streamRequest(streamRequest, requestContext, Messages.toStreamCallback(callback, true)); + streamRequest(streamRequest, requestContext, Messages.toStreamCallback(callback, addContentLengthHeader)); } @Override diff --git a/r2-core/src/test/java/com/linkedin/r2/transport/common/TestAbstractClient.java b/r2-core/src/test/java/com/linkedin/r2/transport/common/TestAbstractClient.java new file mode 100644 index 0000000000..c8416e50ea --- /dev/null +++ b/r2-core/src/test/java/com/linkedin/r2/transport/common/TestAbstractClient.java @@ -0,0 +1,64 @@ +package com.linkedin.r2.transport.common; + +import com.linkedin.common.callback.Callback; +import com.linkedin.common.callback.FutureCallback; +import com.linkedin.common.util.None; +import com.linkedin.data.ByteString; +import com.linkedin.r2.message.RequestContext; +import com.linkedin.r2.message.rest.RestRequest; +import com.linkedin.r2.message.rest.RestRequestBuilder; +import com.linkedin.r2.message.rest.RestResponse; +import com.linkedin.r2.message.stream.StreamRequest; +import com.linkedin.r2.message.stream.StreamResponse; +import com.linkedin.r2.message.stream.StreamResponseBuilder; +import com.linkedin.r2.message.stream.entitystream.ByteStringWriter; +import com.linkedin.r2.message.stream.entitystream.EntityStreams; +import java.net.URI; +import java.util.concurrent.TimeUnit; +import org.junit.Assert; +import org.junit.Test; + + +public class TestAbstractClient { + public static final String URI = "http://localhost:8080/"; + public static final String RESPONSE_DATA = "This is not empty"; + private static final String CONTENT_LENGTH = "Content-Length"; + private static final String GET_HTTP_METHOD = "GET"; + private static final String HEAD_HTTP_METHOD = "HEAD"; + + @Test + public void testHeaderIsNotOverriddenForHEADRequests() throws Exception { + ConcreteClient concreteClient = new ConcreteClient(); + + // Assert that proper content-length is set with non HEADER requests + RestRequest restRequest = new RestRequestBuilder(new URI(URI)).setMethod(GET_HTTP_METHOD).build(); + FutureCallback restResponseCallback = new FutureCallback<>(); + concreteClient.restRequest(restRequest, new RequestContext(), restResponseCallback); + RestResponse response = restResponseCallback.get(10, TimeUnit.SECONDS); + Assert.assertNotNull(response); + Assert.assertTrue(response.getHeaders().containsKey(CONTENT_LENGTH)); + Assert.assertEquals(Integer.parseInt(response.getHeader(CONTENT_LENGTH)), RESPONSE_DATA.length()); + + // Assert that existing content-length is not overridden for HEADER requests + restRequest = new RestRequestBuilder(new URI(URI)).setMethod(HEAD_HTTP_METHOD).build(); + restResponseCallback = new FutureCallback<>(); + concreteClient.restRequest(restRequest, new RequestContext(), restResponseCallback); + response = restResponseCallback.get(10, TimeUnit.SECONDS); + Assert.assertNotNull(response); + Assert.assertFalse(response.getHeaders().containsKey(CONTENT_LENGTH)); + } + + static class ConcreteClient extends AbstractClient { + @Override + public void shutdown(Callback callback) { + + } + + @Override + public void streamRequest(StreamRequest request, RequestContext requestContext, Callback callback) { + StreamResponse response = new StreamResponseBuilder().build( + EntityStreams.newEntityStream(new ByteStringWriter(ByteString.copy(RESPONSE_DATA.getBytes())))); + callback.onSuccess(response); + } + } +} \ No newline at end of file diff --git a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java index 03161458d1..5a4caddd45 100644 --- a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java +++ b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java @@ -62,6 +62,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; import javax.net.ssl.SSLContext; @@ -1041,6 +1042,7 @@ public Object[][] parametersProvider() { * @param isFullRequest Whether to buffer a full request before stream * @throws Exception */ + @Ignore("Test is too flaky and HttpNettyStreamClient is no longer used after enabling PipelineV2") @Test(dataProvider = "requestResponseParameters", retryAnalyzer = ThreeRetries.class) public void testStreamRequests( AbstractNettyStreamClient client, diff --git a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java index 1a53dd1dfd..9f59d8c767 100644 --- a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java +++ b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java @@ -54,6 +54,7 @@ import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; @@ -124,6 +125,7 @@ public void testMaxConcurrentStreamExhaustion() throws Exception * When a request fails due to {@link TimeoutException}, connection should not be destroyed. * @throws Exception */ + @Ignore("Test is too flaky and Http2NettyStreamClient is no longer used after enabling PipelineV2") @Test(timeOut = TEST_TIMEOUT) public void testChannelReusedAfterRequestTimeout() throws Exception { diff --git a/restli-tools/build.gradle b/restli-tools/build.gradle index 05805febdb..6ff1ad8198 100644 --- a/restli-tools/build.gradle +++ b/restli-tools/build.gradle @@ -1,3 +1,41 @@ +plugins { + id "java-library" +} + +//This block is only supported and required when building with JDK11+ +if (JavaVersion.current() >= JavaVersion.VERSION_11) { + //We need a custom source set for JDK11 only classes + sourceSets { + java11 { + java { + srcDirs = ['src/main/java11'] + } + } + } + //This compile task is automatically generated by java-library plugin for custom JDK11 only source set + //We need to explicitly set code versions and override defaults + compileJava11Java { + sourceCompatibility = 11 + targetCompatibility = 11 + options.compilerArgs.addAll(['--release', '11']) + } + + jar { + //We package JDK11+ classes into a custom folder. + //JVM will load the class if version of the class is equal or less than version of JVM. + //Thus JDK8 or JDK9 will load default class from "com" folder and JDK11+ will load the custom folder + into('META-INF/versions/11') { + from sourceSets.java11.output + } + manifest { + attributes( + "Manifest-Version": "1.0", + "Multi-Release": true + ) + } + } +} + dependencies { compile project(':data') compile project(':r2-core') @@ -22,6 +60,33 @@ dependencies { testCompile externalDependency.junit testCompile externalDependency.commonsHttpClient testCompile externalDependency.javaparser + + if (JavaVersion.current() >= JavaVersion.VERSION_11) { + //Custom dependency set is required for JDK11+ only source set + java11Implementation files(sourceSets.main.output.classesDirs) + java11Compile project(':data') + java11Compile project(':r2-core') + java11Compile project(':li-jersey-uri') + java11Compile project(':generator') + java11Compile project(':pegasus-common') + java11Compile project(':restli-common') + java11Compile project(':restli-client') + java11Compile project(':restli-server') + java11Compile externalDependency.caffeine + java11Compile externalDependency.commonsIo + java11Compile externalDependency.codemodel + java11Compile externalDependency.commonsCli + java11Compile externalDependency.commonsLang + java11Compile externalDependency.jacksonCore + java11Compile externalDependency.jacksonDataBind + java11Compile externalDependency.velocity + + java11Compile externalDependency.mockito + java11Compile externalDependency.testng + java11Compile externalDependency.junit + java11Compile externalDependency.commonsHttpClient + java11Compile externalDependency.javaparser + } } apply from: "${buildScriptDirPath}/restModel.gradle" diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java index f0cd6ecc41..c5049f72c4 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java @@ -60,9 +60,9 @@ public class DocletDocsProvider implements DocsProvider private RestLiDoclet _doclet; public DocletDocsProvider(String apiName, - String[] classpath, - String[] sourcePaths, - String[] resourcePackages) + String[] classpath, + String[] sourcePaths, + String[] resourcePackages) { _apiName = apiName; _classpath = classpath; @@ -108,10 +108,10 @@ public void registerSourceFiles(Collection sourceFileNames) } _doclet = RestLiDoclet.generateDoclet(_apiName, - sysoutWriter, - nullWriter, - nullWriter, - javadocArgs.toArray(new String[0])); + sysoutWriter, + nullWriter, + nullWriter, + javadocArgs.toArray(new String[0])); } @Override @@ -270,4 +270,4 @@ private static boolean isActionParamAnnotation(AnnotationDesc annotationDesc) { return ActionParam.class.getCanonicalName().equals(annotationDesc.annotationType().qualifiedName()); } -} +} \ No newline at end of file diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java index 025765a017..1ae6cdc753 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java @@ -71,10 +71,10 @@ public class RestLiDoclet * @throws IllegalArgumentException if Javadoc fails to generate docs. */ public static synchronized RestLiDoclet generateDoclet(String programName, - PrintWriter errWriter, - PrintWriter warnWriter, - PrintWriter noticeWriter, - String[] args) + PrintWriter errWriter, + PrintWriter warnWriter, + PrintWriter noticeWriter, + String[] args) { final int javadocRetCode = Main.execute(programName, errWriter, warnWriter, noticeWriter, RestLiDoclet.class.getName(), args); if (javadocRetCode != 0) @@ -237,4 +237,4 @@ public boolean equals(Object obj) private final String _methodQualifiedName; private final List _parameterTypeNames; } -} +} \ No newline at end of file diff --git a/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java new file mode 100644 index 0000000000..5817e33b0e --- /dev/null +++ b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java @@ -0,0 +1,295 @@ +/* + Copyright (c) 2012 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.restli.tools.idlgen; + + +import com.linkedin.restli.internal.server.model.ResourceModelEncoder.DocsProvider; +import com.linkedin.restli.server.annotations.ActionParam; +import com.linkedin.restli.server.annotations.QueryParam; + +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.FileVisitResult; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.sun.source.doctree.DocTree; +import com.sun.source.doctree.ReturnTree; +import com.sun.source.doctree.UnknownBlockTagTree; +import org.apache.commons.io.output.NullWriter; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.lang.model.element.AnnotationMirror; +import javax.lang.model.element.AnnotationValue; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.element.VariableElement; + + +/** + * Specialized {@link DocsProvider} whose documentation comes from the Javadoc Doclet {@link RestLiDoclet}. + * + * @author dellamag + */ +public class DocletDocsProvider implements DocsProvider +{ + private static final Logger log = LoggerFactory.getLogger(DocletDocsProvider.class); + + private final String _apiName; + private final String[] _classpath; + private final String[] _sourcePaths; + private final String[] _resourcePackages; + + private RestLiDoclet _doclet; + + public DocletDocsProvider(String apiName, + String[] classpath, + String[] sourcePaths, + String[] resourcePackages) + { + _apiName = apiName; + _classpath = classpath; + _sourcePaths = sourcePaths; + _resourcePackages = resourcePackages; + } + + @Override + public Set supportedFileExtensions() + { + return Collections.singleton(".java"); + } + + /** + * Recursively collect all Java file paths under the sourcePaths if packageNames is null or empty. Else, only + * collect the Java file paths whose package name starts with packageNames. + * + * @param sourcePaths source paths to be queried + * @param packageNames target package names to be matched + * @return list of Java file paths + */ + public static List collectSourceFiles(List sourcePaths, List packageNames) throws IOException { + List sourceFiles = new ArrayList<>(); + for (String sourcePath : sourcePaths) { + Path basePath = Paths.get(sourcePath); + if (!Files.exists(basePath)) { + continue; + } + Files.walkFileTree(basePath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().endsWith(".java")) { + if (packageNames == null || packageNames.isEmpty()) { + sourceFiles.add(file.toString()); + } else { + String packageName = basePath.relativize(file.getParent()).toString().replace('/', '.'); + for (String targetPackageName : packageNames) { + if (packageName.startsWith(targetPackageName)) { + sourceFiles.add(file.toString()); + break; + } + } + } + } + return FileVisitResult.CONTINUE; + } + }); + } + return sourceFiles; + } + + @Override + public void registerSourceFiles(Collection sourceFileNames) + { + log.debug("Executing Javadoc tool..."); + final String flatClasspath; + if (_classpath == null) + { + flatClasspath = System.getProperty("java.class.path"); + } + else + { + flatClasspath = StringUtils.join(_classpath, ":"); + } + + final PrintWriter sysoutWriter = new PrintWriter(System.out, true); + final PrintWriter nullWriter = new PrintWriter(new NullWriter()); + + + List sourceFiles; + try + { + sourceFiles = collectSourceFiles(Arrays.asList(_sourcePaths), + _resourcePackages == null ? null : Arrays.asList(_resourcePackages)); + } + catch (IOException e) + { + throw new RuntimeException("Failed to collect source files", e); + } + + _doclet = RestLiDoclet.generateDoclet(_apiName, + sysoutWriter, + nullWriter, + nullWriter, + flatClasspath, + sourceFiles + ); + } + + @Override + public String getClassDoc(Class resourceClass) + { + final TypeElement doc = _doclet.getClassDoc(resourceClass); + if (doc == null) + { + return null; + } + return buildDoc(_doclet.getDocCommentStrForElement(doc)); + } + + public String getClassDeprecatedTag(Class resourceClass) { + TypeElement typeElement = _doclet.getClassDoc(resourceClass); + if (typeElement == null) { + return null; + } + return formatDeprecatedTags(typeElement); + } + + private String formatDeprecatedTags(Element element) { + List deprecatedTags = _doclet.getDeprecatedTags(element); + if (!deprecatedTags.isEmpty()) { + StringBuilder deprecatedText = new StringBuilder(); + for (int i = 0; i < deprecatedTags.size(); i++) { + deprecatedText.append(deprecatedTags.get(i)); + if (i < deprecatedTags.size() - 1) { + deprecatedText.append(" "); + } + } + return deprecatedText.toString(); + } else { + return null; + } + } + @Override + public String getMethodDoc(Method method) + { + final ExecutableElement doc = _doclet.getMethodDoc(method); + if (doc == null) + { + return null; + } + + return buildDoc(_doclet.getDocCommentStrForElement(doc)); + } + + @Override + public String getMethodDeprecatedTag(Method method) + { + final ExecutableElement doc = _doclet.getMethodDoc(method); + if (doc == null) + { + return null; + } + + return formatDeprecatedTags(doc); + } + + + @Override + public String getParamDoc(Method method, String name) + { + final ExecutableElement methodDoc = _doclet.getMethodDoc(method); + + if (methodDoc == null) + { + return null; + } + Map paramTags = _doclet.getParamTags(methodDoc); + for (VariableElement parameter : methodDoc.getParameters()) + { + for (AnnotationMirror annotationMirror : parameter.getAnnotationMirrors()) + { + if (isQueryParamAnnotation(annotationMirror) || isActionParamAnnotation(annotationMirror)) + { + for (Map.Entry entry : annotationMirror.getElementValues().entrySet()) + { + if ("value".equals(entry.getKey().getSimpleName().toString()) && name.equals(entry.getValue().getValue())) + { + return paramTags.get(parameter.getSimpleName().toString()); + } + } + } + } + } + + return null; + } + + @Override + public String getReturnDoc(Method method) + { + ExecutableElement methodElement = _doclet.getMethodDoc(method); + if (methodElement != null) { + for (DocTree docTree : _doclet.getDocCommentTreeForMethod(method).getBlockTags()) { + if (!docTree.toString().toLowerCase().startsWith("@return")) { + continue; + } + DocTree.Kind kind = docTree.getKind(); + if (kind == DocTree.Kind.RETURN) { + ReturnTree returnTree = (ReturnTree) docTree; + return buildDoc(returnTree.getDescription().toString()); + } else if (kind == DocTree.Kind.UNKNOWN_BLOCK_TAG) { + UnknownBlockTagTree unknownBlockTagTree = (UnknownBlockTagTree) docTree; + return buildDoc(unknownBlockTagTree.getContent().toString()); + } + } + } + return null; + } + + private static String buildDoc(String docText) + { + if (docText != null && !docText.isEmpty()) + { + return DocletHelper.processDocCommentStr(docText); + } + return null; + } + + private static boolean isQueryParamAnnotation(AnnotationMirror annotationMirror) + { + return QueryParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); + } + + private static boolean isActionParamAnnotation(AnnotationMirror annotationMirror) + { + return ActionParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); + } +} \ No newline at end of file diff --git a/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletHelper.java b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletHelper.java new file mode 100644 index 0000000000..5581034aa4 --- /dev/null +++ b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletHelper.java @@ -0,0 +1,77 @@ +/* + Copyright (c) 2012 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.restli.tools.idlgen; + +import com.sun.source.doctree.DocCommentTree; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Helper class that defines generic util methods related to {@link jdk.javadoc.doclet.Doclet}. + * + * @author Yan Zhou + */ +public class DocletHelper { + /** + * Get the canonical name of the inputTypeStr, which does not include any reference to its formal type parameter + * when it comes to generic type. For example, the canonical name of the interface java.util.Set is java.util.Set. + * + * @param inputTypeStr class/method/variable type str + * @return canonical name of the inputTypeStr + */ + public static String getCanonicalName(String inputTypeStr) { + if (inputTypeStr == null) { + return null; + } + Pattern pattern = Pattern.compile("<.*>"); + Matcher matcher = pattern.matcher(inputTypeStr); + StringBuilder sb = new StringBuilder(); + int start = 0; + while (matcher.find()) { + sb.append(inputTypeStr.substring(start, matcher.start())); + start = matcher.end(); + } + sb.append(inputTypeStr.substring(start)); + return sb.toString(); + } + + /** + * When {@link DocCommentTree} return Java Doc comment string, they wrap certain chars with commas. For example, + *

will become ,

, This method serves to remove such redundant commas if any. + * + * @param inputCommentStr input Java Doc comment string generated by {@link DocCommentTree} + * @return processed string with redundant commas removed + */ + public static String processDocCommentStr(String inputCommentStr) { + if (inputCommentStr == null) { + return null; + } + Pattern pattern = Pattern.compile("(\\,)(<.*>|\\{@.*\\}|>|<)(\\,)?"); + Matcher matcher = pattern.matcher(inputCommentStr); + StringBuilder sb = new StringBuilder(); + int start = 0; + while (matcher.find()) { + sb.append(inputCommentStr.substring(start, matcher.start())); + int end = matcher.group(3) == null ? matcher.end() : matcher.end() - 1; + sb.append(inputCommentStr.substring(matcher.start() + 1, end).replace(",", "")); + start = matcher.end(); + } + sb.append(inputCommentStr.substring(start)); + return sb.toString(); + } +} \ No newline at end of file diff --git a/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/RestLiDoclet.java b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/RestLiDoclet.java new file mode 100644 index 0000000000..a50824e26e --- /dev/null +++ b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/RestLiDoclet.java @@ -0,0 +1,405 @@ +/* + Copyright (c) 2012 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.restli.tools.idlgen; + + +import com.sun.source.doctree.DocCommentTree; +import com.sun.source.doctree.DocTree; +import com.sun.source.doctree.ParamTree; +import com.sun.source.doctree.DeprecatedTree; +import jdk.javadoc.doclet.Doclet; +import jdk.javadoc.doclet.DocletEnvironment; +import jdk.javadoc.doclet.Reporter; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.VariableElement; +import javax.lang.model.type.TypeMirror; +import javax.tools.DocumentationTool; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; +import java.io.PrintWriter; +import java.lang.reflect.Method; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * Custom Javadoc processor that merges documentation into the restspec.json. The embedded Javadoc + * generator is basically a commandline tool wrapper and it runs in complete isolation from the rest + * of the application. Due to the fact that the Javadoc tool instantiates RestLiDoclet, we cannot + * cleanly integrate the output into the {@link RestLiResourceModelExporter} tool. Thus, we're just + * dumping the docs into a static Map which can be accessed by {@link RestLiResourceModelExporter}. + * + * This class supports multiple runs of Javadoc Doclet API {@link DocumentationTool}. + * Each run will be assigned an unique "Doclet ID", returned by + * {@link #generateDoclet(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String, List)}. + * The Doclet ID should be subsequently used to initialize {@link DocletDocsProvider}. + * + * This class is thread-safe. However, #generateJavadoc() will be synchronized. + * + * @author dellamag + */ +public class RestLiDoclet implements Doclet +{ + private static RestLiDoclet _currentDocLet = null; + private final DocInfo _docInfo; + private final DocletEnvironment _docEnv; + + /** + * Generate Javadoc and return the generated RestLiDoclet instance. + * This method is synchronized. + * + * @param programName Name of the program (for error messages). + * @param errWriter PrintWriter to receive error messages. + * @param warnWriter PrintWriter to receive warning messages. + * @param noticeWriter PrintWriter to receive notice messages. + * @param flatClassPath Flat path to classes to be used. + * @param sourceFiles List of Java source files to be analyzed. + * @return the generated RestLiDoclet instance. + * @throws IllegalArgumentException if Javadoc fails to generate docs. + */ + public static synchronized RestLiDoclet generateDoclet(String programName, + PrintWriter errWriter, + PrintWriter warnWriter, + PrintWriter noticeWriter, + String flatClassPath, + List sourceFiles + ) + { + noticeWriter.println("Generating Javadoc for " + programName); + + DocumentationTool docTool = ToolProvider.getSystemDocumentationTool(); + StandardJavaFileManager fileManager = docTool.getStandardFileManager(null, null, null); + Iterable fileObjects = fileManager.getJavaFileObjectsFromPaths( + sourceFiles.stream().map(Paths::get).collect(Collectors.toList())); + + // Set up the Javadoc task options + List taskOptions = new ArrayList<>(); + taskOptions.add("-classpath"); + taskOptions.add(flatClassPath); + + // Create and run the Javadoc task + DocumentationTool.DocumentationTask task = docTool.getTask(errWriter, + fileManager, diagnostic -> { + switch (diagnostic.getKind()) { + case ERROR: + errWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case WARNING: + warnWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case NOTE: + noticeWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + } + }, + RestLiDoclet.class, + taskOptions, + fileObjects); + + boolean success = task.call(); + if (!success) + { + throw new IllegalArgumentException("Javadoc generation failed"); + } + + return _currentDocLet; + } + + /** + * Entry point for Javadoc Doclet. + * + * @param docEnv {@link DocletEnvironment} passed in by Javadoc + * @return is successful or not + */ + @Override + public boolean run(DocletEnvironment docEnv) { + final DocInfo docInfo = new DocInfo(); + + // Iterate through the TypeElements (class and interface declarations) + for (Element element : docEnv.getIncludedElements()) { + if (element instanceof TypeElement) { + TypeElement typeElement = (TypeElement) element; + docInfo.setClassDoc(typeElement.getQualifiedName().toString(), typeElement); + + // Iterate through the methods of the TypeElement + for (Element enclosedElement : typeElement.getEnclosedElements()) { + if (enclosedElement instanceof ExecutableElement) { + ExecutableElement methodElement = (ExecutableElement) enclosedElement; + docInfo.setMethodDoc(MethodIdentity.create(methodElement), methodElement); + } + } + } + } + + _currentDocLet = new RestLiDoclet(docInfo, docEnv); + + return true; + } + + @Override + public void init(Locale locale, Reporter reporter) { + // no-ops + } + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + + @Override + public Set getSupportedOptions() { + return Set.of(); + } + + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latest(); + } + + private RestLiDoclet(DocInfo docInfo, DocletEnvironment docEnv) + { + _docInfo = docInfo; + _docEnv = docEnv; + } + + /** + * The reason why we create a public empty constructor is because JavadocTaskImpl in JDK 11 requires it when using reflection. + * Otherwise, there will be NoSuchMethodException: com.linkedin.restli.tools.idlgen.RestLiDoclet.() + */ + public RestLiDoclet() { + _docInfo = null; + _docEnv = null; + } + + /** + * Query Javadoc {@link TypeElement} for the specified resource class. + * + * @param resourceClass resource class to be queried + * @return corresponding {@link TypeElement} + */ + public TypeElement getClassDoc(Class resourceClass) + { + return _docInfo.getClassDoc(resourceClass.getCanonicalName()); + } + + /** + * Query Javadoc {@link ExecutableElement} for the specified Java method. + * + * @param method Java method to be queried + * @return corresponding {@link ExecutableElement} + */ + public ExecutableElement getMethodDoc(Method method) + { + final MethodIdentity methodId = MethodIdentity.create(method); + return _docInfo.getMethodDoc(methodId); + } + + private static class DocInfo + { + public TypeElement getClassDoc(String className) { + return _classNameToClassDoc.get(className); + } + + public ExecutableElement getMethodDoc(MethodIdentity methodId) { + return _methodIdToMethodDoc.get(methodId); + } + + public void setClassDoc(String className, TypeElement classDoc) { + _classNameToClassDoc.put(className, classDoc); + } + + public void setMethodDoc(MethodIdentity methodId, ExecutableElement methodDoc) { + _methodIdToMethodDoc.put(methodId, methodDoc); + } + + private final Map _classNameToClassDoc = new HashMap<>(); + private final Map _methodIdToMethodDoc = new HashMap<>(); + } + + private static class MethodIdentity + { + public static MethodIdentity create(Method method) + { + final List parameterTypeNames = new ArrayList<>(); + + // type parameters are not included in identity because of differences between reflection and Doclet: + // e.g. for Collection: + // reflection Type.toString() -> Collection + // Doclet Type.toString() -> Collection + for (Class paramClass: method.getParameterTypes()) + { + parameterTypeNames.add(paramClass.getCanonicalName()); + } + + return new MethodIdentity(method.getDeclaringClass().getName() + "." + method.getName(), parameterTypeNames); + } + + public static MethodIdentity create(ExecutableElement method) + { + final List parameterTypeNames = new ArrayList<>(); + for (VariableElement param : method.getParameters()) { + TypeMirror type = param.asType(); + parameterTypeNames.add(DocletHelper.getCanonicalName(type.toString())); + } + + return new MethodIdentity(method.getEnclosingElement().toString() + "." + method.getSimpleName().toString(), + parameterTypeNames); + } + + private MethodIdentity(String methodQualifiedName, List parameterTypeNames) + { + _methodQualifiedName = methodQualifiedName; + _parameterTypeNames = parameterTypeNames; + } + + @Override + public int hashCode() + { + return new HashCodeBuilder(17, 29). + append(_methodQualifiedName). + append(_parameterTypeNames). + toHashCode(); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + { + return true; + } + + if (obj == null) + { + return false; + } + + if (getClass() != obj.getClass()) + { + return false; + } + + final MethodIdentity other = (MethodIdentity) obj; + return new EqualsBuilder(). + append(_methodQualifiedName, other._methodQualifiedName). + append(_parameterTypeNames, other._parameterTypeNames). + isEquals(); + } + + private final String _methodQualifiedName; + private final List _parameterTypeNames; + } + + /** + * Get the list of deprecated tags for the specified element. + * + * @param element {@link Element} to be queried + * @return list of deprecated tags for the specified element + */ + public List getDeprecatedTags(Element element) { + List deprecatedTags = new ArrayList<>(); + DocCommentTree docCommentTree = getDocCommentTreeForElement(element); + if (docCommentTree == null) { + return deprecatedTags; + } + for (DocTree docTree :docCommentTree.getBlockTags()) { + if (docTree.getKind() == DocTree.Kind.DEPRECATED) { + DeprecatedTree deprecatedTree = (DeprecatedTree) docTree; + String deprecatedComment = deprecatedTree.getBody().toString(); + deprecatedTags.add(deprecatedComment); + } + } + return deprecatedTags; + } + + /** + * Get the map from param name to param comment for the specified executableElement. + * + * @param executableElement {@link ExecutableElement} to be queried + * @return map from param name to param comment for the specified executableElement + */ + public Map getParamTags(ExecutableElement executableElement) { + Map paramTags = new HashMap<>(); + DocCommentTree docCommentTree = getDocCommentTreeForElement(executableElement); + if (docCommentTree == null) { + return paramTags; + } + for (DocTree docTree : docCommentTree.getBlockTags()) { + if (docTree.getKind() == DocTree.Kind.PARAM) { + ParamTree paramTree = (ParamTree) docTree; + String paramName = paramTree.getName().toString(); + String paramComment = paramTree.getDescription().toString(); + if (paramComment != null) { + paramTags.put(paramName, paramComment); + } + } + } + return paramTags; + } + + /** + * Get the {@link DocCommentTree} for the specified element. + * + * @param element {@link Element} to be queried + * @return {@link DocCommentTree} for the specified element + */ + public DocCommentTree getDocCommentTreeForElement(Element element) { + return element == null ? null : _docEnv.getDocTrees().getDocCommentTree(element); + } + + /** + * Get the Doc Comment string for the specified element. + * + * @param element {@link Element} to be queried + * @return Doc Comment string for the specified element + */ + public String getDocCommentStrForElement(Element element) { + DocCommentTree docCommentTree = getDocCommentTreeForElement(element); + return docCommentTree == null ? null : docCommentTree.getFullBody().toString(); + } + + /** + * Get the {@link DocCommentTree} for the specified method. + * + * @param method {@link Method} to be queried + * @return {@link DocCommentTree} for the specified method + */ + public DocCommentTree getDocCommentTreeForMethod(Method method) { + TypeElement typeElement = getClassDoc(method.getDeclaringClass()); + if (typeElement == null) { + return null; + } + for (Element element : typeElement.getEnclosedElements()) { + if (element.getSimpleName().toString().equals(method.getName())) { + return getDocCommentTreeForElement(element); + } + } + return null; + } +} \ No newline at end of file