From 8d632d44388fbc06f3aace0562cc81381161304e Mon Sep 17 00:00:00 2001 From: Yan Zhou Date: Mon, 8 May 2023 10:35:48 -0700 Subject: [PATCH 1/5] Rewrite Java Doc logic with JDK 17 compatible APIs --- build.gradle | 2 +- .../PegasusPluginCacheabilityTest.groovy | 2 +- .../PegasusPluginIntegrationTest.groovy | 6 +- .../tools/idlgen/DocletDocsProvider.java | 212 +++++++------ .../restli/tools/idlgen/DocletHelper.java | 77 +++++ .../restli/tools/idlgen/RestLiDoclet.java | 283 ++++++++++++++---- .../restli/tools/idlgen/TestDocletHelper.java | 108 +++++++ 7 files changed, 531 insertions(+), 159 deletions(-) create mode 100644 restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletHelper.java create mode 100644 restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java diff --git a/build.gradle b/build.gradle index 3fab2df6f5..cfe88a689a 100644 --- a/build.gradle +++ b/build.gradle @@ -144,7 +144,7 @@ allprojects { throw new GradleScriptException("Pegasus required Java 8 or later to build, current version: ${JavaVersion.current()}", null) } // for all supported versions that we test build, fail the build if any compilation warnings are reported - compile.options.compilerArgs = ['-Xlint', '-Xlint:-path', '-Xlint:-static', '-Werror'] + compile.options.compilerArgs = ['-Xlint', '-Xlint:-path', '-Xlint:-static'] } tasks.withType(Javadoc) diff --git a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy index dad8526c8d..2c4926798e 100644 --- a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy +++ b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginCacheabilityTest.groovy @@ -96,6 +96,6 @@ class PegasusPluginCacheabilityTest extends Specification { preparedSchema.exists() where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2' ] } } diff --git a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy index ea6002f4ea..bbd1dca7f3 100644 --- a/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy +++ b/gradle-plugins/src/integTest/groovy/com/linkedin/pegasus/gradle/PegasusPluginIntegrationTest.groovy @@ -34,7 +34,7 @@ class PegasusPluginIntegrationTest extends Specification { result.task(':mainDataTemplateJar').outcome == SUCCESS where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] } @Unroll @@ -102,7 +102,7 @@ class PegasusPluginIntegrationTest extends Specification { assertZipContains(dataTemplateArtifact, 'extensions/com/linkedin/LatLongExtensions.pdl') where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] } def 'mainCopySchema task will remove stale PDSC'() { @@ -262,7 +262,7 @@ class PegasusPluginIntegrationTest extends Specification { result.task(':impl:compileJava').outcome == SUCCESS where: - gradleVersion << [ '4.0', '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] + gradleVersion << [ '5.2.1', '5.6.4', '6.9', '7.0.2', '7.5.1' ] } private static boolean assertZipContains(File zip, String path) { diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java index f0cd6ecc41..5817e33b0e 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java @@ -21,27 +21,38 @@ import com.linkedin.restli.server.annotations.ActionParam; import com.linkedin.restli.server.annotations.QueryParam; +import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.FileVisitResult; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; -import com.sun.javadoc.AnnotationDesc; -import com.sun.javadoc.ClassDoc; -import com.sun.javadoc.Doc; -import com.sun.javadoc.MethodDoc; -import com.sun.javadoc.ParamTag; -import com.sun.javadoc.Parameter; -import com.sun.javadoc.Tag; +import com.sun.source.doctree.DocTree; +import com.sun.source.doctree.ReturnTree; +import com.sun.source.doctree.UnknownBlockTagTree; import org.apache.commons.io.output.NullWriter; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.lang.model.element.AnnotationMirror; +import javax.lang.model.element.AnnotationValue; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.element.VariableElement; + /** * Specialized {@link DocsProvider} whose documentation comes from the Javadoc Doclet {@link RestLiDoclet}. @@ -60,9 +71,9 @@ public class DocletDocsProvider implements DocsProvider private RestLiDoclet _doclet; public DocletDocsProvider(String apiName, - String[] classpath, - String[] sourcePaths, - String[] resourcePackages) + String[] classpath, + String[] sourcePaths, + String[] resourcePackages) { _apiName = apiName; _classpath = classpath; @@ -76,11 +87,48 @@ public Set supportedFileExtensions() return Collections.singleton(".java"); } + /** + * Recursively collect all Java file paths under the sourcePaths if packageNames is null or empty. Else, only + * collect the Java file paths whose package name starts with packageNames. + * + * @param sourcePaths source paths to be queried + * @param packageNames target package names to be matched + * @return list of Java file paths + */ + public static List collectSourceFiles(List sourcePaths, List packageNames) throws IOException { + List sourceFiles = new ArrayList<>(); + for (String sourcePath : sourcePaths) { + Path basePath = Paths.get(sourcePath); + if (!Files.exists(basePath)) { + continue; + } + Files.walkFileTree(basePath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().endsWith(".java")) { + if (packageNames == null || packageNames.isEmpty()) { + sourceFiles.add(file.toString()); + } else { + String packageName = basePath.relativize(file.getParent()).toString().replace('/', '.'); + for (String targetPackageName : packageNames) { + if (packageName.startsWith(targetPackageName)) { + sourceFiles.add(file.toString()); + break; + } + } + } + } + return FileVisitResult.CONTINUE; + } + }); + } + return sourceFiles; + } + @Override public void registerSourceFiles(Collection sourceFileNames) { log.debug("Executing Javadoc tool..."); - final String flatClasspath; if (_classpath == null) { @@ -93,89 +141,78 @@ public void registerSourceFiles(Collection sourceFileNames) final PrintWriter sysoutWriter = new PrintWriter(System.out, true); final PrintWriter nullWriter = new PrintWriter(new NullWriter()); - final List javadocArgs = new ArrayList<>(Arrays.asList("-classpath", - flatClasspath, - "-sourcepath", - StringUtils.join(_sourcePaths, ":"))); - if (_resourcePackages != null) + + + List sourceFiles; + try { - javadocArgs.add("-subpackages"); - javadocArgs.add(StringUtils.join(_resourcePackages, ":")); + sourceFiles = collectSourceFiles(Arrays.asList(_sourcePaths), + _resourcePackages == null ? null : Arrays.asList(_resourcePackages)); } - else + catch (IOException e) { - javadocArgs.addAll(sourceFileNames); + throw new RuntimeException("Failed to collect source files", e); } _doclet = RestLiDoclet.generateDoclet(_apiName, - sysoutWriter, - nullWriter, - nullWriter, - javadocArgs.toArray(new String[0])); + sysoutWriter, + nullWriter, + nullWriter, + flatClasspath, + sourceFiles + ); } @Override public String getClassDoc(Class resourceClass) { - final ClassDoc doc = _doclet.getClassDoc(resourceClass); + final TypeElement doc = _doclet.getClassDoc(resourceClass); if (doc == null) { return null; } - - return buildDoc(doc.commentText()); + return buildDoc(_doclet.getDocCommentStrForElement(doc)); } - @Override - public String getClassDeprecatedTag(Class resourceClass) - { - final ClassDoc doc = _doclet.getClassDoc(resourceClass); - if (doc == null) - { + public String getClassDeprecatedTag(Class resourceClass) { + TypeElement typeElement = _doclet.getClassDoc(resourceClass); + if (typeElement == null) { return null; } - - return formatDeprecatedTags(doc); + return formatDeprecatedTags(typeElement); } - private static String formatDeprecatedTags(Doc doc) - { - Tag[] deprecatedTags = doc.tags("deprecated"); - if(deprecatedTags.length > 0) - { + private String formatDeprecatedTags(Element element) { + List deprecatedTags = _doclet.getDeprecatedTags(element); + if (!deprecatedTags.isEmpty()) { StringBuilder deprecatedText = new StringBuilder(); - for(int i = 0; i < deprecatedTags.length; i++) - { - deprecatedText.append(deprecatedTags[i].text()); - if(i < deprecatedTags.length - 1) - { + for (int i = 0; i < deprecatedTags.size(); i++) { + deprecatedText.append(deprecatedTags.get(i)); + if (i < deprecatedTags.size() - 1) { deprecatedText.append(" "); } } return deprecatedText.toString(); - } - else - { + } else { return null; } } - @Override public String getMethodDoc(Method method) { - final MethodDoc doc = _doclet.getMethodDoc(method); + final ExecutableElement doc = _doclet.getMethodDoc(method); if (doc == null) { return null; } - return buildDoc(doc.commentText()); + return buildDoc(_doclet.getDocCommentStrForElement(doc)); } @Override public String getMethodDeprecatedTag(Method method) { - final MethodDoc doc = _doclet.getMethodDoc(method); + final ExecutableElement doc = _doclet.getMethodDoc(method); if (doc == null) { return null; @@ -184,32 +221,28 @@ public String getMethodDeprecatedTag(Method method) return formatDeprecatedTags(doc); } + @Override public String getParamDoc(Method method, String name) { - final MethodDoc methodDoc = _doclet.getMethodDoc(method); + final ExecutableElement methodDoc = _doclet.getMethodDoc(method); if (methodDoc == null) { return null; } - - for (Parameter parameter : methodDoc.parameters()) + Map paramTags = _doclet.getParamTags(methodDoc); + for (VariableElement parameter : methodDoc.getParameters()) { - for (AnnotationDesc annotationDesc : parameter.annotations()) + for (AnnotationMirror annotationMirror : parameter.getAnnotationMirrors()) { - if (annotationDesc.isSynthesized()) + if (isQueryParamAnnotation(annotationMirror) || isActionParamAnnotation(annotationMirror)) { - continue; - } - - if (isQueryParamAnnotation(annotationDesc) || isActionParamAnnotation(annotationDesc)) - { - for (AnnotationDesc.ElementValuePair pair : annotationDesc.elementValues()) + for (Map.Entry entry : annotationMirror.getElementValues().entrySet()) { - if ("value".equals(pair.element().name()) && name.equals(pair.value().value())) + if ("value".equals(entry.getKey().getSimpleName().toString()) && name.equals(entry.getValue().getValue())) { - return getParamTagDoc(methodDoc, parameter.name()); + return paramTags.get(parameter.getSimpleName().toString()); } } } @@ -219,34 +252,25 @@ public String getParamDoc(Method method, String name) return null; } - private static String getParamTagDoc(MethodDoc methodDoc, String name) - { - for (ParamTag tag : methodDoc.paramTags()) - { - if (name.equals(tag.parameterName())) - { - return buildDoc(tag.parameterComment()); - } - } - - return null; - } - @Override public String getReturnDoc(Method method) { - final MethodDoc methodDoc = _doclet.getMethodDoc(method); - if (methodDoc != null) - { - for (Tag tag : methodDoc.tags()) - { - if(tag.name().toLowerCase().equals("@return")) - { - return buildDoc(tag.text()); + ExecutableElement methodElement = _doclet.getMethodDoc(method); + if (methodElement != null) { + for (DocTree docTree : _doclet.getDocCommentTreeForMethod(method).getBlockTags()) { + if (!docTree.toString().toLowerCase().startsWith("@return")) { + continue; + } + DocTree.Kind kind = docTree.getKind(); + if (kind == DocTree.Kind.RETURN) { + ReturnTree returnTree = (ReturnTree) docTree; + return buildDoc(returnTree.getDescription().toString()); + } else if (kind == DocTree.Kind.UNKNOWN_BLOCK_TAG) { + UnknownBlockTagTree unknownBlockTagTree = (UnknownBlockTagTree) docTree; + return buildDoc(unknownBlockTagTree.getContent().toString()); } } } - return null; } @@ -254,20 +278,18 @@ private static String buildDoc(String docText) { if (docText != null && !docText.isEmpty()) { - return docText; + return DocletHelper.processDocCommentStr(docText); } - return null; } - - private static boolean isQueryParamAnnotation(AnnotationDesc annotationDesc) + private static boolean isQueryParamAnnotation(AnnotationMirror annotationMirror) { - return QueryParam.class.getCanonicalName().equals(annotationDesc.annotationType().qualifiedName()); + return QueryParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); } - private static boolean isActionParamAnnotation(AnnotationDesc annotationDesc) + private static boolean isActionParamAnnotation(AnnotationMirror annotationMirror) { - return ActionParam.class.getCanonicalName().equals(annotationDesc.annotationType().qualifiedName()); + return ActionParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); } -} +} \ No newline at end of file diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletHelper.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletHelper.java new file mode 100644 index 0000000000..5581034aa4 --- /dev/null +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletHelper.java @@ -0,0 +1,77 @@ +/* + Copyright (c) 2012 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.restli.tools.idlgen; + +import com.sun.source.doctree.DocCommentTree; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Helper class that defines generic util methods related to {@link jdk.javadoc.doclet.Doclet}. + * + * @author Yan Zhou + */ +public class DocletHelper { + /** + * Get the canonical name of the inputTypeStr, which does not include any reference to its formal type parameter + * when it comes to generic type. For example, the canonical name of the interface java.util.Set is java.util.Set. + * + * @param inputTypeStr class/method/variable type str + * @return canonical name of the inputTypeStr + */ + public static String getCanonicalName(String inputTypeStr) { + if (inputTypeStr == null) { + return null; + } + Pattern pattern = Pattern.compile("<.*>"); + Matcher matcher = pattern.matcher(inputTypeStr); + StringBuilder sb = new StringBuilder(); + int start = 0; + while (matcher.find()) { + sb.append(inputTypeStr.substring(start, matcher.start())); + start = matcher.end(); + } + sb.append(inputTypeStr.substring(start)); + return sb.toString(); + } + + /** + * When {@link DocCommentTree} return Java Doc comment string, they wrap certain chars with commas. For example, + *

will become ,

, This method serves to remove such redundant commas if any. + * + * @param inputCommentStr input Java Doc comment string generated by {@link DocCommentTree} + * @return processed string with redundant commas removed + */ + public static String processDocCommentStr(String inputCommentStr) { + if (inputCommentStr == null) { + return null; + } + Pattern pattern = Pattern.compile("(\\,)(<.*>|\\{@.*\\}|>|<)(\\,)?"); + Matcher matcher = pattern.matcher(inputCommentStr); + StringBuilder sb = new StringBuilder(); + int start = 0; + while (matcher.find()) { + sb.append(inputCommentStr.substring(start, matcher.start())); + int end = matcher.group(3) == null ? matcher.end() : matcher.end() - 1; + sb.append(inputCommentStr.substring(matcher.start() + 1, end).replace(",", "")); + start = matcher.end(); + } + sb.append(inputCommentStr.substring(start)); + return sb.toString(); + } +} \ No newline at end of file diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java index 025765a017..a50824e26e 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java @@ -17,21 +17,35 @@ package com.linkedin.restli.tools.idlgen; -import com.sun.javadoc.ClassDoc; -import com.sun.javadoc.MethodDoc; -import com.sun.javadoc.Parameter; -import com.sun.javadoc.RootDoc; -import com.sun.javadoc.Type; -import com.sun.tools.javadoc.Main; +import com.sun.source.doctree.DocCommentTree; +import com.sun.source.doctree.DocTree; +import com.sun.source.doctree.ParamTree; +import com.sun.source.doctree.DeprecatedTree; +import jdk.javadoc.doclet.Doclet; +import jdk.javadoc.doclet.DocletEnvironment; +import jdk.javadoc.doclet.Reporter; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; - +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.VariableElement; +import javax.lang.model.type.TypeMirror; +import javax.tools.DocumentationTool; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; import java.io.PrintWriter; import java.lang.reflect.Method; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /** @@ -41,45 +55,77 @@ * cleanly integrate the output into the {@link RestLiResourceModelExporter} tool. Thus, we're just * dumping the docs into a static Map which can be accessed by {@link RestLiResourceModelExporter}. * - * This class supports multiple runs of Javadoc Doclet API {@link Main#execute(String[])}. + * This class supports multiple runs of Javadoc Doclet API {@link DocumentationTool}. * Each run will be assigned an unique "Doclet ID", returned by - * {@link #generateDoclet(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String[])}. + * {@link #generateDoclet(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String, List)}. * The Doclet ID should be subsequently used to initialize {@link DocletDocsProvider}. * * This class is thread-safe. However, #generateJavadoc() will be synchronized. * * @author dellamag - * @see Main#execute(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String, String[]) */ -public class RestLiDoclet +public class RestLiDoclet implements Doclet { private static RestLiDoclet _currentDocLet = null; - private final DocInfo _docInfo; + private final DocletEnvironment _docEnv; /** - * Generate Javadoc and return associated Doclet ID. + * Generate Javadoc and return the generated RestLiDoclet instance. * This method is synchronized. * * @param programName Name of the program (for error messages). * @param errWriter PrintWriter to receive error messages. * @param warnWriter PrintWriter to receive warning messages. * @param noticeWriter PrintWriter to receive notice messages. - * @param args The command line parameters. - * @return an unique doclet ID which represent the subsequent Main#execute() run. - * @throws IllegalStateException if the generated doclet ID is already used. Try again. + * @param flatClassPath Flat path to classes to be used. + * @param sourceFiles List of Java source files to be analyzed. + * @return the generated RestLiDoclet instance. * @throws IllegalArgumentException if Javadoc fails to generate docs. */ public static synchronized RestLiDoclet generateDoclet(String programName, - PrintWriter errWriter, - PrintWriter warnWriter, - PrintWriter noticeWriter, - String[] args) + PrintWriter errWriter, + PrintWriter warnWriter, + PrintWriter noticeWriter, + String flatClassPath, + List sourceFiles + ) { - final int javadocRetCode = Main.execute(programName, errWriter, warnWriter, noticeWriter, RestLiDoclet.class.getName(), args); - if (javadocRetCode != 0) + noticeWriter.println("Generating Javadoc for " + programName); + + DocumentationTool docTool = ToolProvider.getSystemDocumentationTool(); + StandardJavaFileManager fileManager = docTool.getStandardFileManager(null, null, null); + Iterable fileObjects = fileManager.getJavaFileObjectsFromPaths( + sourceFiles.stream().map(Paths::get).collect(Collectors.toList())); + + // Set up the Javadoc task options + List taskOptions = new ArrayList<>(); + taskOptions.add("-classpath"); + taskOptions.add(flatClassPath); + + // Create and run the Javadoc task + DocumentationTool.DocumentationTask task = docTool.getTask(errWriter, + fileManager, diagnostic -> { + switch (diagnostic.getKind()) { + case ERROR: + errWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case WARNING: + warnWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case NOTE: + noticeWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + } + }, + RestLiDoclet.class, + taskOptions, + fileObjects); + + boolean success = task.call(); + if (!success) { - throw new IllegalArgumentException("Javadoc failed with return code " + javadocRetCode); + throw new IllegalArgumentException("Javadoc generation failed"); } return _currentDocLet; @@ -88,80 +134,112 @@ public static synchronized RestLiDoclet generateDoclet(String programName, /** * Entry point for Javadoc Doclet. * - * @param root {@link RootDoc} passed in by Javadoc + * @param docEnv {@link DocletEnvironment} passed in by Javadoc * @return is successful or not */ - public static boolean start(RootDoc root) - { + @Override + public boolean run(DocletEnvironment docEnv) { final DocInfo docInfo = new DocInfo(); - for (ClassDoc classDoc : root.classes()) - { - docInfo.setClassDoc(classDoc.qualifiedName(), classDoc); - - for (MethodDoc methodDoc : classDoc.methods()) - { - docInfo.setMethodDoc(MethodIdentity.create(methodDoc), methodDoc); + // Iterate through the TypeElements (class and interface declarations) + for (Element element : docEnv.getIncludedElements()) { + if (element instanceof TypeElement) { + TypeElement typeElement = (TypeElement) element; + docInfo.setClassDoc(typeElement.getQualifiedName().toString(), typeElement); + + // Iterate through the methods of the TypeElement + for (Element enclosedElement : typeElement.getEnclosedElements()) { + if (enclosedElement instanceof ExecutableElement) { + ExecutableElement methodElement = (ExecutableElement) enclosedElement; + docInfo.setMethodDoc(MethodIdentity.create(methodElement), methodElement); + } + } } } - _currentDocLet = new RestLiDoclet(docInfo); + _currentDocLet = new RestLiDoclet(docInfo, docEnv); return true; } - private RestLiDoclet(DocInfo docInfo) + @Override + public void init(Locale locale, Reporter reporter) { + // no-ops + } + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + + @Override + public Set getSupportedOptions() { + return Set.of(); + } + + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latest(); + } + + private RestLiDoclet(DocInfo docInfo, DocletEnvironment docEnv) { _docInfo = docInfo; + _docEnv = docEnv; } /** - * Query Javadoc {@link ClassDoc} for the specified resource class. + * The reason why we create a public empty constructor is because JavadocTaskImpl in JDK 11 requires it when using reflection. + * Otherwise, there will be NoSuchMethodException: com.linkedin.restli.tools.idlgen.RestLiDoclet.() + */ + public RestLiDoclet() { + _docInfo = null; + _docEnv = null; + } + + /** + * Query Javadoc {@link TypeElement} for the specified resource class. * * @param resourceClass resource class to be queried - * @return corresponding {@link ClassDoc} + * @return corresponding {@link TypeElement} */ - public ClassDoc getClassDoc(Class resourceClass) + public TypeElement getClassDoc(Class resourceClass) { return _docInfo.getClassDoc(resourceClass.getCanonicalName()); } /** - * Query Javadoc {@link MethodDoc} for the specified Java method. + * Query Javadoc {@link ExecutableElement} for the specified Java method. * * @param method Java method to be queried - * @return corresponding {@link MethodDoc} + * @return corresponding {@link ExecutableElement} */ - public MethodDoc getMethodDoc(Method method) + public ExecutableElement getMethodDoc(Method method) { final MethodIdentity methodId = MethodIdentity.create(method); - return _docInfo.getMethodDoc(methodId); + return _docInfo.getMethodDoc(methodId); } private static class DocInfo { - public ClassDoc getClassDoc(String className) - { + public TypeElement getClassDoc(String className) { return _classNameToClassDoc.get(className); } - public MethodDoc getMethodDoc(MethodIdentity methodId) - { + public ExecutableElement getMethodDoc(MethodIdentity methodId) { return _methodIdToMethodDoc.get(methodId); } - public void setClassDoc(String className, ClassDoc classDoc) - { + public void setClassDoc(String className, TypeElement classDoc) { _classNameToClassDoc.put(className, classDoc); } - public void setMethodDoc(MethodIdentity methodId, MethodDoc methodDoc) - { + public void setMethodDoc(MethodIdentity methodId, ExecutableElement methodDoc) { _methodIdToMethodDoc.put(methodId, methodDoc); } - private final Map _classNameToClassDoc = new HashMap<>(); - private final Map _methodIdToMethodDoc = new HashMap<>(); + private final Map _classNameToClassDoc = new HashMap<>(); + private final Map _methodIdToMethodDoc = new HashMap<>(); } private static class MethodIdentity @@ -182,16 +260,16 @@ public static MethodIdentity create(Method method) return new MethodIdentity(method.getDeclaringClass().getName() + "." + method.getName(), parameterTypeNames); } - public static MethodIdentity create(MethodDoc method) + public static MethodIdentity create(ExecutableElement method) { final List parameterTypeNames = new ArrayList<>(); - for (Parameter param: method.parameters()) - { - Type type = param.type(); - parameterTypeNames.add(type.qualifiedTypeName() + type.dimension()); + for (VariableElement param : method.getParameters()) { + TypeMirror type = param.asType(); + parameterTypeNames.add(DocletHelper.getCanonicalName(type.toString())); } - return new MethodIdentity(method.qualifiedName(), parameterTypeNames); + return new MethodIdentity(method.getEnclosingElement().toString() + "." + method.getSimpleName().toString(), + parameterTypeNames); } private MethodIdentity(String methodQualifiedName, List parameterTypeNames) @@ -237,4 +315,91 @@ public boolean equals(Object obj) private final String _methodQualifiedName; private final List _parameterTypeNames; } -} + + /** + * Get the list of deprecated tags for the specified element. + * + * @param element {@link Element} to be queried + * @return list of deprecated tags for the specified element + */ + public List getDeprecatedTags(Element element) { + List deprecatedTags = new ArrayList<>(); + DocCommentTree docCommentTree = getDocCommentTreeForElement(element); + if (docCommentTree == null) { + return deprecatedTags; + } + for (DocTree docTree :docCommentTree.getBlockTags()) { + if (docTree.getKind() == DocTree.Kind.DEPRECATED) { + DeprecatedTree deprecatedTree = (DeprecatedTree) docTree; + String deprecatedComment = deprecatedTree.getBody().toString(); + deprecatedTags.add(deprecatedComment); + } + } + return deprecatedTags; + } + + /** + * Get the map from param name to param comment for the specified executableElement. + * + * @param executableElement {@link ExecutableElement} to be queried + * @return map from param name to param comment for the specified executableElement + */ + public Map getParamTags(ExecutableElement executableElement) { + Map paramTags = new HashMap<>(); + DocCommentTree docCommentTree = getDocCommentTreeForElement(executableElement); + if (docCommentTree == null) { + return paramTags; + } + for (DocTree docTree : docCommentTree.getBlockTags()) { + if (docTree.getKind() == DocTree.Kind.PARAM) { + ParamTree paramTree = (ParamTree) docTree; + String paramName = paramTree.getName().toString(); + String paramComment = paramTree.getDescription().toString(); + if (paramComment != null) { + paramTags.put(paramName, paramComment); + } + } + } + return paramTags; + } + + /** + * Get the {@link DocCommentTree} for the specified element. + * + * @param element {@link Element} to be queried + * @return {@link DocCommentTree} for the specified element + */ + public DocCommentTree getDocCommentTreeForElement(Element element) { + return element == null ? null : _docEnv.getDocTrees().getDocCommentTree(element); + } + + /** + * Get the Doc Comment string for the specified element. + * + * @param element {@link Element} to be queried + * @return Doc Comment string for the specified element + */ + public String getDocCommentStrForElement(Element element) { + DocCommentTree docCommentTree = getDocCommentTreeForElement(element); + return docCommentTree == null ? null : docCommentTree.getFullBody().toString(); + } + + /** + * Get the {@link DocCommentTree} for the specified method. + * + * @param method {@link Method} to be queried + * @return {@link DocCommentTree} for the specified method + */ + public DocCommentTree getDocCommentTreeForMethod(Method method) { + TypeElement typeElement = getClassDoc(method.getDeclaringClass()); + if (typeElement == null) { + return null; + } + for (Element element : typeElement.getEnclosedElements()) { + if (element.getSimpleName().toString().equals(method.getName())) { + return getDocCommentTreeForElement(element); + } + } + return null; + } +} \ No newline at end of file diff --git a/restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java b/restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java new file mode 100644 index 0000000000..9ec837a348 --- /dev/null +++ b/restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java @@ -0,0 +1,108 @@ +/* + Copyright (c) 2012 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.restli.tools.idlgen; + + +import org.testng.Assert; +import org.testng.annotations.Test; + + +/** + * Tests to ensure that util methods in {@link DocletHelper} work correctly. + * + * @author Yan Zhou + */ +public class TestDocletHelper +{ + @Test + public void testGetCanonicalName() { + // input is null, so no op + String input = null; + String actualOutput = DocletHelper.getCanonicalName(input); + String expectedOutput = null; + Assert.assertTrue(actualOutput == expectedOutput); + + // input does not match the pattern, so no op + input = "asdfljk sdf \n * "; + actualOutput = DocletHelper.getCanonicalName(input); + expectedOutput = input; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches the pattern i.e. <*>, so relevant substring is removed + input = "))?? )* * "; + actualOutput = DocletHelper.getCanonicalName(input); + expectedOutput = "))?? )* * "; + Assert.assertEquals(actualOutput, expectedOutput); + } + + @Test + public void testProcessDocCommentStr() { + // input is null, so no op + String input = null; + String actualOutput = DocletHelper.processDocCommentStr(input); + String expectedOutput = null; + Assert.assertTrue(actualOutput == expectedOutput); + + // input does not match the pattern, so no op + input = "asdf \n () , ? * "; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = input; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches pattern, i.e. ,{@xxx }, so redundant commas are removed + input = " ,{@xxx }, "; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = " {@xxx } "; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches the pattern, i.e. ,{@xxx } so redundant commas are removed + input = " ,{@xxx } "; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = " {@xxx } "; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches, i.e. ,>, so redundant commas are removed + input = " ? ,>, ,"; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = " ? > ,"; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches the pattern, i.e. ,<, so redundant commas are removed + input = " ? ,<, ,"; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = " ? < ,"; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches the pattern, i.e. ,< so redundant commas are removed + input = " ? ,< ,"; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = " ? < ,"; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches the pattern, i.e. ,<,>, so redundant commas are removed + input = " ,<,>, ? *(,"; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = " <> ? *(,"; + Assert.assertEquals(actualOutput, expectedOutput); + + // input matches the pattern, i.e. ,, so redundant commas are removed + input = " ,,,?"; + actualOutput = DocletHelper.processDocCommentStr(input); + expectedOutput = " ,?"; + Assert.assertEquals(actualOutput, expectedOutput); + } +} From b03568a980459fbaf47c951263978ee75f5e7638 Mon Sep 17 00:00:00 2001 From: John Stewart Date: Tue, 2 May 2023 15:05:11 -0700 Subject: [PATCH 2/5] HEAD requests content-length is not based on the body as per the http spec. Fixing it to not override the content-length if the request is HEADER type. --- CHANGELOG.md | 7 +- gradle.properties | 2 +- .../r2/transport/common/AbstractClient.java | 5 +- .../transport/common/TestAbstractClient.java | 64 +++++++++++++++++++ .../client/TestHttpNettyStreamClient.java | 2 + .../http2/TestHttp2NettyStreamClient.java | 2 + 6 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 r2-core/src/test/java/com/linkedin/r2/transport/common/TestAbstractClient.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b42d3beb8..58ea79a3f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.42.0] - 2023-05-02 +- Remove the overriding of content-length for HEADER requests as per HTTP Spec + More details about this issue can be found @ https://jira01.corp.linkedin.com:8443/browse/SI-31814 + ## [29.41.12] - 2023-04-06 - Introduce `@extension.injectedUrnParts` ER annotation. - This will be used as the replacement for using `@extension.params` to specify injected URN parts. @@ -5458,7 +5462,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.41.12...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.42.0...master +[29.42.0]: https://github.com/linkedin/rest.li/compare/v29.41.12...v29.42.0 [29.41.12]: https://github.com/linkedin/rest.li/compare/v29.41.11...v29.41.12 [29.41.11]: https://github.com/linkedin/rest.li/compare/v29.41.10...v29.41.11 [29.41.10]: https://github.com/linkedin/rest.li/compare/v29.41.9...v29.41.10 diff --git a/gradle.properties b/gradle.properties index 9b6ec08860..ce14fea603 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.41.12 +version=29.42.0 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true diff --git a/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java b/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java index 2b5396962f..ebab27f33e 100644 --- a/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java +++ b/r2-core/src/main/java/com/linkedin/r2/transport/common/AbstractClient.java @@ -54,6 +54,7 @@ */ public abstract class AbstractClient implements Client { + public static final String HTTP_HEAD_METHOD = "HEAD"; @Override public Future restRequest(RestRequest request) @@ -88,8 +89,10 @@ public void restRequest(RestRequest request, RequestContext requestContext, Call // IS_FULL_REQUEST flag, if set true, would result in the request being sent without using chunked transfer encoding // This is needed as the legacy R2 server (before 2.8.0) does not support chunked transfer encoding. requestContext.putLocalAttr(R2Constants.IS_FULL_REQUEST, true); + + boolean addContentLengthHeader = !HTTP_HEAD_METHOD.equalsIgnoreCase(request.getMethod()); // here we add back the content-length header for the response because some client code depends on this header - streamRequest(streamRequest, requestContext, Messages.toStreamCallback(callback, true)); + streamRequest(streamRequest, requestContext, Messages.toStreamCallback(callback, addContentLengthHeader)); } @Override diff --git a/r2-core/src/test/java/com/linkedin/r2/transport/common/TestAbstractClient.java b/r2-core/src/test/java/com/linkedin/r2/transport/common/TestAbstractClient.java new file mode 100644 index 0000000000..c8416e50ea --- /dev/null +++ b/r2-core/src/test/java/com/linkedin/r2/transport/common/TestAbstractClient.java @@ -0,0 +1,64 @@ +package com.linkedin.r2.transport.common; + +import com.linkedin.common.callback.Callback; +import com.linkedin.common.callback.FutureCallback; +import com.linkedin.common.util.None; +import com.linkedin.data.ByteString; +import com.linkedin.r2.message.RequestContext; +import com.linkedin.r2.message.rest.RestRequest; +import com.linkedin.r2.message.rest.RestRequestBuilder; +import com.linkedin.r2.message.rest.RestResponse; +import com.linkedin.r2.message.stream.StreamRequest; +import com.linkedin.r2.message.stream.StreamResponse; +import com.linkedin.r2.message.stream.StreamResponseBuilder; +import com.linkedin.r2.message.stream.entitystream.ByteStringWriter; +import com.linkedin.r2.message.stream.entitystream.EntityStreams; +import java.net.URI; +import java.util.concurrent.TimeUnit; +import org.junit.Assert; +import org.junit.Test; + + +public class TestAbstractClient { + public static final String URI = "http://localhost:8080/"; + public static final String RESPONSE_DATA = "This is not empty"; + private static final String CONTENT_LENGTH = "Content-Length"; + private static final String GET_HTTP_METHOD = "GET"; + private static final String HEAD_HTTP_METHOD = "HEAD"; + + @Test + public void testHeaderIsNotOverriddenForHEADRequests() throws Exception { + ConcreteClient concreteClient = new ConcreteClient(); + + // Assert that proper content-length is set with non HEADER requests + RestRequest restRequest = new RestRequestBuilder(new URI(URI)).setMethod(GET_HTTP_METHOD).build(); + FutureCallback restResponseCallback = new FutureCallback<>(); + concreteClient.restRequest(restRequest, new RequestContext(), restResponseCallback); + RestResponse response = restResponseCallback.get(10, TimeUnit.SECONDS); + Assert.assertNotNull(response); + Assert.assertTrue(response.getHeaders().containsKey(CONTENT_LENGTH)); + Assert.assertEquals(Integer.parseInt(response.getHeader(CONTENT_LENGTH)), RESPONSE_DATA.length()); + + // Assert that existing content-length is not overridden for HEADER requests + restRequest = new RestRequestBuilder(new URI(URI)).setMethod(HEAD_HTTP_METHOD).build(); + restResponseCallback = new FutureCallback<>(); + concreteClient.restRequest(restRequest, new RequestContext(), restResponseCallback); + response = restResponseCallback.get(10, TimeUnit.SECONDS); + Assert.assertNotNull(response); + Assert.assertFalse(response.getHeaders().containsKey(CONTENT_LENGTH)); + } + + static class ConcreteClient extends AbstractClient { + @Override + public void shutdown(Callback callback) { + + } + + @Override + public void streamRequest(StreamRequest request, RequestContext requestContext, Callback callback) { + StreamResponse response = new StreamResponseBuilder().build( + EntityStreams.newEntityStream(new ByteStringWriter(ByteString.copy(RESPONSE_DATA.getBytes())))); + callback.onSuccess(response); + } + } +} \ No newline at end of file diff --git a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java index 03161458d1..5a4caddd45 100644 --- a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java +++ b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/TestHttpNettyStreamClient.java @@ -62,6 +62,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; import javax.net.ssl.SSLContext; @@ -1041,6 +1042,7 @@ public Object[][] parametersProvider() { * @param isFullRequest Whether to buffer a full request before stream * @throws Exception */ + @Ignore("Test is too flaky and HttpNettyStreamClient is no longer used after enabling PipelineV2") @Test(dataProvider = "requestResponseParameters", retryAnalyzer = ThreeRetries.class) public void testStreamRequests( AbstractNettyStreamClient client, diff --git a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java index 1a53dd1dfd..9f59d8c767 100644 --- a/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java +++ b/r2-netty/src/test/java/com/linkedin/r2/transport/http/client/stream/http2/TestHttp2NettyStreamClient.java @@ -54,6 +54,7 @@ import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Ignore; import org.testng.annotations.Test; @@ -124,6 +125,7 @@ public void testMaxConcurrentStreamExhaustion() throws Exception * When a request fails due to {@link TimeoutException}, connection should not be destroyed. * @throws Exception */ + @Ignore("Test is too flaky and Http2NettyStreamClient is no longer used after enabling PipelineV2") @Test(timeOut = TEST_TIMEOUT) public void testChannelReusedAfterRequestTimeout() throws Exception { From bcd4102b88160da20558a9e9be564dbfd4b6c55c Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Fri, 12 May 2023 09:46:06 -0400 Subject: [PATCH 3/5] Get line/col number in PDL schema encoder (#895) --- CHANGELOG.md | 6 +- .../data/schema/SchemaToPdlEncoder.java | 99 +++++++- .../linkedin/util/LineColumnNumberWriter.java | 226 ++++++++++++++++++ .../util/TestLineColumnNumberWriter.java | 51 ++++ .../generator/test/pdl/PdlEncoderTest.java | 73 ++++++ gradle.properties | 2 +- 6 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 data/src/main/java/com/linkedin/util/LineColumnNumberWriter.java create mode 100644 data/src/test/java/com/linkedin/util/TestLineColumnNumberWriter.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 58ea79a3f9..6aa2d2a7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.42.1] - 2023-05-11 +- Add support for returning location of schema elements from the PDL schema encoder. + ## [29.42.0] - 2023-05-02 - Remove the overriding of content-length for HEADER requests as per HTTP Spec More details about this issue can be found @ https://jira01.corp.linkedin.com:8443/browse/SI-31814 @@ -5462,7 +5465,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.42.0...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.42.1...master +[29.42.1]: https://github.com/linkedin/rest.li/compare/v29.42.0...v29.42.1 [29.42.0]: https://github.com/linkedin/rest.li/compare/v29.41.12...v29.42.0 [29.41.12]: https://github.com/linkedin/rest.li/compare/v29.41.11...v29.41.12 [29.41.11]: https://github.com/linkedin/rest.li/compare/v29.41.10...v29.41.11 diff --git a/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java b/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java index a7f0bca7ad..7d2c04f8c7 100644 --- a/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java +++ b/data/src/main/java/com/linkedin/data/schema/SchemaToPdlEncoder.java @@ -18,11 +18,14 @@ import com.linkedin.data.DataList; import com.linkedin.data.DataMap; +import com.linkedin.data.schema.grammar.PdlSchemaParser; +import com.linkedin.util.LineColumnNumberWriter; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.util.Collections; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -107,15 +110,45 @@ PdlBuilder newBuilderInstance(Writer writer) private String _namespace = ""; private String _package = ""; + private final boolean _trackWriteLocations; + + private final Map _writeLocations; + /** * Construct a .pdl source code encoder. + * The encoding style defaults to {@link EncodingStyle#INDENTED} but may be changed by calling + * {@link #setEncodingStyle(EncodingStyle)}. * * @param out provides the encoded .pdl destination. */ public SchemaToPdlEncoder(Writer out) { - _writer = out; - _encodingStyle = EncodingStyle.INDENTED; + this(out, false); + } + + /** + * Construct a .pdl source code encoder with the option to track line/column of schema elements during writing. + * The encoding style defaults to {@link EncodingStyle#INDENTED} but may be changed by calling + * {@link #setEncodingStyle(EncodingStyle)}. + * + * @param out provides the encoded .pdl destination. + * @param returnContextLocations Enable recording the context locations of schema elements during parsing. The + * locations can be retrieved using {@link #getWriteLocations()} after parsing. + */ + public SchemaToPdlEncoder(Writer out, boolean returnContextLocations) + { + if (returnContextLocations) + { + _writeLocations = new IdentityHashMap<>(); + // Wrap the Writer to track line/column numbers to report to elementWriteListener + _writer = new LineColumnNumberWriter(out); + } else + { + _writer = out; + _writeLocations = Collections.emptyMap(); + } + setEncodingStyle(EncodingStyle.INDENTED); + _trackWriteLocations = returnContextLocations; } /** @@ -126,6 +159,18 @@ public SchemaToPdlEncoder(Writer out) public void setEncodingStyle(EncodingStyle encodingStyle) { _encodingStyle = encodingStyle; + + // When counting column numbers, CompactPDLBuilder treats ',' as whitespace + if (_writer instanceof LineColumnNumberWriter) + { + if (_encodingStyle == EncodingStyle.COMPACT) + { + ((LineColumnNumberWriter) _writer).setIsWhitespaceFunction(c -> Character.isWhitespace(c) || c == ','); + } else + { + ((LineColumnNumberWriter) _writer).setIsWhitespaceFunction(Character::isWhitespace); + } + } } /** @@ -150,10 +195,12 @@ public void encode(DataSchema schema) throws IOException { if (hasNamespace) { + markSchemaElementStartLocation(); _builder.write("namespace") .writeSpace() .writeIdentifier(namedSchema.getNamespace()) .newline(); + recordSchemaElementLocation(namedSchema.getNamespace()); _namespace = namedSchema.getNamespace(); } if (hasPackage) @@ -220,12 +267,14 @@ private void writeInlineSchema(DataSchema schema) throws IOException .increaseIndent(); if (hasNamespaceOverride) { + markSchemaElementStartLocation(); _builder .indent() .write("namespace") .writeSpace() .writeIdentifier(namedSchema.getNamespace()) .newline(); + recordSchemaElementLocation(namedSchema.getNamespace()); _namespace = namedSchema.getNamespace(); } if (hasPackageOverride) @@ -291,8 +340,14 @@ private void writeInlineSchema(DataSchema schema) throws IOException } } + public Map getWriteLocations() + { + return _writeLocations; + } + private void writeRecord(RecordDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("record") .writeSpace() @@ -327,6 +382,7 @@ private void writeRecord(RecordDataSchema schema) throws IOException { writeIncludes(schema, includes); } + recordSchemaElementLocation(schema); } /** @@ -335,6 +391,7 @@ private void writeRecord(RecordDataSchema schema) throws IOException */ private void writeField(RecordDataSchema.Field field) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(field); _builder.indent() .writeIdentifier(field.getName()) @@ -353,6 +410,7 @@ private void writeField(RecordDataSchema.Field field) throws IOException .writeSpace() .writeJson(field.getDefault(), field.getType()); } + recordSchemaElementLocation(field); _builder.newline(); } @@ -382,6 +440,7 @@ private void writeEnum(EnumDataSchema schema) throws IOException DataSchemaConstants.DEPRECATED_SYMBOLS_KEY, properties.get(DataSchemaConstants.DEPRECATED_SYMBOLS_KEY)); + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("enum") .writeSpace() @@ -395,6 +454,7 @@ private void writeEnum(EnumDataSchema schema) throws IOException for (String symbol : schema.getSymbols()) { + markSchemaElementStartLocation(); String docString = docs.get(symbol); DataMap symbolProperties = coercePropertyToDataMapOrFail(schema, DataSchemaConstants.SYMBOL_PROPERTIES_KEY + "." + symbol, @@ -414,24 +474,29 @@ private void writeEnum(EnumDataSchema schema) throws IOException _builder.indent() .writeIdentifier(symbol) .newline(); + recordSchemaElementLocation(symbol); } _builder.decreaseIndent() .indent() .write("}"); + recordSchemaElementLocation(schema); } private void writeFixed(FixedDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("fixed") .writeSpace() .writeIdentifier(schema.getName()) .writeSpace() .write(String.valueOf(schema.getSize())); + recordSchemaElementLocation(schema); } private void writeTyperef(TyperefDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeDocAndProperties(schema); _builder.write("typeref") .writeSpace() @@ -441,24 +506,29 @@ private void writeTyperef(TyperefDataSchema schema) throws IOException .writeSpace(); DataSchema ref = schema.getRef(); writeReferenceOrInline(ref, schema.isRefDeclaredInline()); + recordSchemaElementLocation(schema); } private void writeMap(MapDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeProperties(schema.getProperties()); _builder.write("map[string") .writeComma() .writeSpace(); writeReferenceOrInline(schema.getValues(), schema.isValuesDeclaredInline()); _builder.write("]"); + recordSchemaElementLocation(schema); } private void writeArray(ArrayDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeProperties(schema.getProperties()); _builder.write("array["); writeReferenceOrInline(schema.getItems(), schema.isItemsDeclaredInline()); _builder.write("]"); + recordSchemaElementLocation(schema); } /** @@ -467,6 +537,7 @@ private void writeArray(ArrayDataSchema schema) throws IOException */ private void writeUnion(UnionDataSchema schema) throws IOException { + markSchemaElementStartLocation(); writeProperties(schema.getProperties()); _builder.write("union["); final boolean useMultilineFormat = schema.areMembersAliased() || schema.getMembers().size() >= UNION_MULTILINE_THRESHOLD; @@ -496,6 +567,7 @@ private void writeUnion(UnionDataSchema schema) throws IOException .indent(); } _builder.write("]"); + recordSchemaElementLocation(schema); } /** @@ -505,6 +577,7 @@ private void writeUnion(UnionDataSchema schema) throws IOException */ private void writeUnionMember(UnionDataSchema.Member member, boolean useMultilineFormat) throws IOException { + markSchemaElementStartLocation(); if (member.hasAlias()) { if (StringUtils.isNotBlank(member.getDoc()) || !member.getProperties().isEmpty() || member.isDeclaredInline()) @@ -524,6 +597,7 @@ else if (useMultilineFormat) _builder.indent(); } writeReferenceOrInline(member.getType(), member.isDeclaredInline()); + recordSchemaElementLocation(member); } private void writePrimitive(PrimitiveDataSchema schema) throws IOException @@ -865,4 +939,25 @@ else if (_namespace.equals(schema.getNamespace()) && !_importsByLocalName.contai _builder.writeIdentifier(schema.getFullName()); } } + + void markSchemaElementStartLocation() + { + if (_trackWriteLocations) + { + ((LineColumnNumberWriter) _writer).saveCurrentPosition(); + } + } + + private void recordSchemaElementLocation(Object schemaElement) + { + if (_trackWriteLocations) + { + LineColumnNumberWriter.CharacterPosition startPosition = ((LineColumnNumberWriter) _writer).popSavedPosition(); + LineColumnNumberWriter.CharacterPosition endPosition = + ((LineColumnNumberWriter) _writer).getLastNonWhitespacePosition(); + _writeLocations.put(schemaElement, + new PdlSchemaParser.ParseLocation(startPosition.getLine(), startPosition.getColumn(), endPosition.getLine(), + endPosition.getColumn())); + } + } } diff --git a/data/src/main/java/com/linkedin/util/LineColumnNumberWriter.java b/data/src/main/java/com/linkedin/util/LineColumnNumberWriter.java new file mode 100644 index 0000000000..1e29d7b21f --- /dev/null +++ b/data/src/main/java/com/linkedin/util/LineColumnNumberWriter.java @@ -0,0 +1,226 @@ +package com.linkedin.util; + +import java.io.IOException; +import java.io.Writer; +import java.util.Objects; +import java.util.Stack; +import java.util.function.Predicate; + + +/** + * Wraps a {@link Writer} and tracks current line and column numbers + */ +public final class LineColumnNumberWriter extends Writer +{ + + private final Writer _writer; + private final Stack _savedPositionStack = new Stack<>(); + private int _column; + private int _line; + private int _previousChar; + private Predicate _isWhitespaceFunction; + private final CharacterPosition _lastNonWhitespacePosition; + + /** + * Creates a new writer. + * + * @param out a Writer object to provide the underlying stream. + */ + public LineColumnNumberWriter(Writer out) + { + _writer = out; + _column = 1; + _line = 1; + _previousChar = -1; + _isWhitespaceFunction = (Character::isWhitespace); + _lastNonWhitespacePosition = new CharacterPosition(0, 0); + } + + /** + * Returns 1 based indices of row and column next character will be written to + */ + public CharacterPosition getCurrentPosition() + { + return new CharacterPosition(_line, _column); + } + + /** + * Returns 1 based indices of last row and column ignoring trailing whitespace characters + */ + public CharacterPosition getLastNonWhitespacePosition() + { + return _lastNonWhitespacePosition; + } + + /** + * Saves current row and column to be retrieved later by calling {@link #popSavedPosition()} + * + * Saved positions are stored in a stack so that calls to saveCurrentPosition() and + * {@link #popSavedPosition()} can be nested. Saved positions are adjusted to skip whitespace to make it + * easier to get actual token start positions in indented output. If you call saveCurrentPosition() at column x + * and then write four spaces followed by non-whitespace, the column number returned by + * {@link #popSavedPosition()} will be x + 4. + */ + public void saveCurrentPosition() + { + _savedPositionStack.push(new CharacterPosition(_line, _column)); + } + + /** + * Retrieves row and column from the last time {@link #saveCurrentPosition()} was called + */ + public CharacterPosition popSavedPosition() + { + return _savedPositionStack.pop(); + } + + /** + * Override definition of whitespace used to adjust character positions to skip + * whitespace. By default, the definition of whitespace is provided by {@link java.lang.Character#isWhitespace} + */ + public void setIsWhitespaceFunction(Predicate isWhitespaceFunction) + { + _isWhitespaceFunction = isWhitespaceFunction; + } + + @Override + public void write(char[] cbuf, int off, int len) throws IOException + { + _writer.write(cbuf, off, len); + for (; len > 0; len--) + { + char c = cbuf[off++]; + int lastLine = _line; + int lastColumn = _column; + updateCurrentPosition(c); + _previousChar = c; + if (_isWhitespaceFunction.test(c)) + { + updateSavedPositionsForWhitespace(lastLine, lastColumn); + } else + { + _lastNonWhitespacePosition.line = lastLine; + _lastNonWhitespacePosition.column = lastColumn; + } + } + } + + @Override + public void flush() throws IOException + { + _writer.flush(); + } + + @Override + public void close() throws IOException + { + _writer.close(); + } + + @Override + public String toString() + { + return _writer.toString(); + } + + private void updateCurrentPosition(char c) + { + if (_previousChar == '\r') + { + if (c == '\n') + { + _column = 1; + } else + { + _column = 2; + } + } else if (c == '\n' || c == '\r') + { + _column = 1; + ++_line; + } else + { + ++_column; + } + } + + /** + * Any saved positions that are equal to the current row and column are set to the current position in order to + * remove leading whitespace. Once the first non-whitespace character is written, the current position will be + * different from any saved positions and the current position will advance. + */ + private void updateSavedPositionsForWhitespace(int lastLine, int lastColumn) + { + for (int i = _savedPositionStack.size() - 1; i >= 0; --i) + { + CharacterPosition savedCharacterPosition = _savedPositionStack.get(i); + if (savedCharacterPosition.line == lastLine && savedCharacterPosition.column == lastColumn) + { + savedCharacterPosition.line = _line; + savedCharacterPosition.column = _column; + } else + { + break; + } + } + } + + /** + * Row and column numbers of a character in Writer output + */ + public static class CharacterPosition + { + + private int line; + private int column; + + CharacterPosition(int line, int column) + { + this.line = line; + this.column = column; + } + + /** + * 1-based index of line in writer output + */ + public int getLine() + { + return line; + } + + /** + * 1-based index of column in writer output + */ + public int getColumn() + { + return column; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + CharacterPosition characterPosition = (CharacterPosition) o; + return line == characterPosition.line && column == characterPosition.column; + } + + @Override + public int hashCode() + { + return Objects.hash(line, column); + } + + @Override + public String toString() + { + return "CharacterPosition{" + "line=" + line + ", column=" + column + '}'; + } + } +} diff --git a/data/src/test/java/com/linkedin/util/TestLineColumnNumberWriter.java b/data/src/test/java/com/linkedin/util/TestLineColumnNumberWriter.java new file mode 100644 index 0000000000..c3010149cc --- /dev/null +++ b/data/src/test/java/com/linkedin/util/TestLineColumnNumberWriter.java @@ -0,0 +1,51 @@ +package com.linkedin.util; + +import java.io.IOException; +import java.io.StringWriter; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class TestLineColumnNumberWriter +{ + + @Test + public void testHandlesDifferentNewlines() throws IOException + { + LineColumnNumberWriter writer = new LineColumnNumberWriter(new StringWriter()); + writer.write("1\n2\n3\n"); + Assert.assertEquals(writer.getCurrentPosition(), new LineColumnNumberWriter.CharacterPosition(4, 1)); + writer.write("1\r\n2\r\n3\r\n"); + Assert.assertEquals(writer.getCurrentPosition(), new LineColumnNumberWriter.CharacterPosition(7, 1)); + writer.write("1\r2\r3\r"); + Assert.assertEquals(writer.getCurrentPosition(), new LineColumnNumberWriter.CharacterPosition(10, 1)); + } + + @Test + public void testSavedPositionIgnoresLeadingWhitespace() throws IOException + { + LineColumnNumberWriter writer = new LineColumnNumberWriter(new StringWriter()); + writer.write("123\n"); + writer.saveCurrentPosition(); + writer.saveCurrentPosition(); + writer.write(" \n "); + writer.write("456"); + writer.saveCurrentPosition(); + writer.write(" 789"); + Assert.assertEquals(writer.popSavedPosition(), new LineColumnNumberWriter.CharacterPosition(3, 8)); + Assert.assertEquals(writer.popSavedPosition(), new LineColumnNumberWriter.CharacterPosition(3, 2)); + Assert.assertEquals(writer.popSavedPosition(), new LineColumnNumberWriter.CharacterPosition(3, 2)); + } + + @Test + public void testGetLastNonWhitespacePosition() throws IOException + { + LineColumnNumberWriter writer = new LineColumnNumberWriter(new StringWriter()); + writer.write("123"); + Assert.assertEquals(writer.getLastNonWhitespacePosition(), new LineColumnNumberWriter.CharacterPosition(1, 3)); + writer.write("\n "); + Assert.assertEquals(writer.getLastNonWhitespacePosition(), new LineColumnNumberWriter.CharacterPosition(1, 3)); + writer.write("4"); + Assert.assertEquals(writer.getLastNonWhitespacePosition(), new LineColumnNumberWriter.CharacterPosition(2, 2)); + } +} diff --git a/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java b/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java index 7d12f35666..ae22fdef75 100644 --- a/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java +++ b/generator-test/src/test/java/com/linkedin/pegasus/generator/test/pdl/PdlEncoderTest.java @@ -21,7 +21,10 @@ import com.linkedin.data.schema.AbstractSchemaParser; import com.linkedin.data.schema.DataSchema; import com.linkedin.data.schema.DataSchemaResolver; +import com.linkedin.data.schema.NamedDataSchema; +import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.schema.SchemaToPdlEncoder; +import com.linkedin.data.schema.UnionDataSchema; import com.linkedin.data.schema.grammar.PdlSchemaParser; import com.linkedin.data.schema.resolver.MultiFormatDataSchemaResolver; import com.linkedin.pegasus.generator.test.idl.EncodingStyle; @@ -31,6 +34,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.StringWriter; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -149,6 +153,75 @@ private void assertRoundTrip(String relativeName) throws IOException } } + @Test(dataProvider = "pdlFilePaths") + public void testTrackWriteLocations(String pdlFilePath) throws IOException + { + assertRoundTripLineColumnNumbersMatch(pdlFilePath); + } + + private void assertRoundTripLineColumnNumbersMatch(String relativeName) throws IOException + { + String fullName = "com.linkedin.pegasus.generator.test.idl." + relativeName; + File file = new File(pegasusSrcDir, "/" + fullName.replace('.', '/') + ".pdl"); + + TypeReferenceFormat referenceFormat = TypeReferenceFormat.PRESERVE; + + // Test all encoding styles + for (SchemaToPdlEncoder.EncodingStyle encodingStyle : SchemaToPdlEncoder.EncodingStyle.values()) + { + String encoded = readAndStandardizeFormat(file, referenceFormat, encodingStyle); + + DataSchemaResolver resolver = MultiFormatDataSchemaResolver.withBuiltinFormats(pegasusSrcDir.getAbsolutePath()); + PdlSchemaParser parser = new PdlSchemaParser(resolver, true); + parser.parse(encoded); + Map parsedLocations = parser.getParseLocations(); + DataSchema parsed = extractSchema(parser, file.getAbsolutePath()); + + StringWriter writer = new StringWriter(); + SchemaToPdlEncoder encoder = new SchemaToPdlEncoder(writer, true); + encoder.setTypeReferenceFormat(referenceFormat); + encoder.setEncodingStyle(encodingStyle); + encoder.encode(parsed); + Map writeLocations = encoder.getWriteLocations(); + + for (Map.Entry expected : parsedLocations.entrySet()) + { + PdlSchemaParser.ParseLocation actual = writeLocations.get(expected.getKey()); + + Assert.assertNotNull(actual, + "Missing location for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getStartLine(), expected.getValue().getStartLine(), + "Start line for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getStartColumn(), expected.getValue().getStartColumn(), + "Start col for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getEndLine(), expected.getValue().getEndLine(), + "End line for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + Assert.assertEquals(actual.getEndColumn(), expected.getValue().getEndColumn(), + "End col for " + expected.getKey() + " in " + file.getAbsolutePath() + ":" + + expected.getValue().getStartLine() + ":" + expected.getValue().getStartColumn()); + } + + Assert.assertEquals(parsedLocations.size(), writeLocations.size(), + "Different numer of element locations for " + file.getAbsolutePath()); + } + } + + private String readAndStandardizeFormat(File file, TypeReferenceFormat typeReferenceFormat, + SchemaToPdlEncoder.EncodingStyle encodingStyle) throws IOException + { + DataSchema parsed = parseSchema(file); + StringWriter writer = new StringWriter(); + SchemaToPdlEncoder encoder = new SchemaToPdlEncoder(writer); + encoder.setEncodingStyle(encodingStyle); + encoder.setTypeReferenceFormat(typeReferenceFormat); + encoder.encode(parsed); + return writer.toString(); + } + private DataSchema parseSchema(File file) throws IOException { DataSchemaResolver resolver = MultiFormatDataSchemaResolver.withBuiltinFormats(pegasusSrcDir.getAbsolutePath()); diff --git a/gradle.properties b/gradle.properties index ce14fea603..3702b1dcc2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.42.0 +version=29.42.1 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true From 34ab538275e09f522717da125dbd3d55409550d9 Mon Sep 17 00:00:00 2001 From: John Stewart Date: Wed, 26 Apr 2023 12:54:21 -0700 Subject: [PATCH 4/5] Improve synchronization on the R2 `RequestContext` local attributes. It is possible to cause a `ConcurrentModificationException` in the `RequestContext` because we synchronize it by using `Collections.synchronizedMap()`, which only synchronizes individual reads/writes. When making a `RequestContext` copy we must iterate over the collection, allowing other threads to interrupt it by modifying the map during the operation. This fix synchronizes the whole copy operation as well. The default mutex used for synchronizing in a `SynchronizedMap` is `this`, so synchronizing on the map itself guards against modification. --- CHANGELOG.md | 6 +++++- gradle.properties | 2 +- .../main/java/com/linkedin/r2/message/RequestContext.java | 8 +++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aa2d2a7c0..e8cc30d9b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.42.2] - 2023-05-11 +- Fix synchronization on `RequestContext` to prevent `ConcurrentModificationException`. + ## [29.42.1] - 2023-05-11 - Add support for returning location of schema elements from the PDL schema encoder. @@ -5465,7 +5468,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.42.1...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.42.2...master +[29.42.2]: https://github.com/linkedin/rest.li/compare/v29.42.1...v29.42.2 [29.42.1]: https://github.com/linkedin/rest.li/compare/v29.42.0...v29.42.1 [29.42.0]: https://github.com/linkedin/rest.li/compare/v29.41.12...v29.42.0 [29.41.12]: https://github.com/linkedin/rest.li/compare/v29.41.11...v29.41.12 diff --git a/gradle.properties b/gradle.properties index 3702b1dcc2..8f88ec9791 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.42.1 +version=29.42.2 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true diff --git a/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java b/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java index 707622e383..717f427352 100644 --- a/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java +++ b/r2-core/src/main/java/com/linkedin/r2/message/RequestContext.java @@ -54,12 +54,14 @@ public RequestContext() */ public RequestContext(RequestContext other) { - _localAttrs = Collections.synchronizedMap(new HashMap<>(other._localAttrs)); + synchronized (other._localAttrs) { + _localAttrs = Collections.synchronizedMap(new HashMap<>(other._localAttrs)); + } } private RequestContext(Map localAttrs) { - _localAttrs = localAttrs; + _localAttrs = Collections.synchronizedMap(localAttrs); } /** @@ -117,7 +119,7 @@ public RequestContext clone() public boolean equals(Object o) { return (o instanceof RequestContext) && - ((RequestContext)o)._localAttrs.equals(this._localAttrs); + ((RequestContext)o)._localAttrs.equals(this._localAttrs); } @Override From fc04976ffdd02e934b99bf871219469aeda55ea3 Mon Sep 17 00:00:00 2001 From: Yan Zhou Date: Thu, 10 Aug 2023 17:35:24 -0700 Subject: [PATCH 5/5] use multi-release jar --- build_script/restModel.gradle | 2 +- restli-tools/build.gradle | 65 +++ .../tools/idlgen/DocletDocsProvider.java | 198 ++++----- .../restli/tools/idlgen/RestLiDoclet.java | 275 +++--------- .../tools/idlgen/DocletDocsProvider.java | 295 +++++++++++++ .../restli/tools/idlgen/DocletHelper.java | 0 .../restli/tools/idlgen/RestLiDoclet.java | 405 ++++++++++++++++++ .../restli/tools/idlgen/TestDocletHelper.java | 108 ----- 8 files changed, 909 insertions(+), 439 deletions(-) create mode 100644 restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java rename restli-tools/src/main/{java => java11}/com/linkedin/restli/tools/idlgen/DocletHelper.java (100%) create mode 100644 restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/RestLiDoclet.java delete mode 100644 restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java diff --git a/build_script/restModel.gradle b/build_script/restModel.gradle index d4a23865fa..9b374408cb 100644 --- a/build_script/restModel.gradle +++ b/build_script/restModel.gradle @@ -79,7 +79,7 @@ project.sourceSets.all { SourceSet sourceSet -> project.tasks[sourceSet.compileJavaTaskName].dependsOn(rootProject.ext.build.restModelGenerateTasks[sourceSet]) } - final Task jarTask = project.tasks[sourceSet.getTaskName('', 'jar')] + final Task jarTask = project.tasks[sourceSet.getName().endsWith('11') ? 'jar' : sourceSet.getTaskName('', 'jar')] jarTask.from(inputParentDirPath) { include "${pegasusDirName}${File.separatorChar}**${File.separatorChar}*.pdsc" include "${pegasusDirName}${File.separatorChar}**${File.separatorChar}*.pdl" diff --git a/restli-tools/build.gradle b/restli-tools/build.gradle index 05805febdb..6ff1ad8198 100644 --- a/restli-tools/build.gradle +++ b/restli-tools/build.gradle @@ -1,3 +1,41 @@ +plugins { + id "java-library" +} + +//This block is only supported and required when building with JDK11+ +if (JavaVersion.current() >= JavaVersion.VERSION_11) { + //We need a custom source set for JDK11 only classes + sourceSets { + java11 { + java { + srcDirs = ['src/main/java11'] + } + } + } + //This compile task is automatically generated by java-library plugin for custom JDK11 only source set + //We need to explicitly set code versions and override defaults + compileJava11Java { + sourceCompatibility = 11 + targetCompatibility = 11 + options.compilerArgs.addAll(['--release', '11']) + } + + jar { + //We package JDK11+ classes into a custom folder. + //JVM will load the class if version of the class is equal or less than version of JVM. + //Thus JDK8 or JDK9 will load default class from "com" folder and JDK11+ will load the custom folder + into('META-INF/versions/11') { + from sourceSets.java11.output + } + manifest { + attributes( + "Manifest-Version": "1.0", + "Multi-Release": true + ) + } + } +} + dependencies { compile project(':data') compile project(':r2-core') @@ -22,6 +60,33 @@ dependencies { testCompile externalDependency.junit testCompile externalDependency.commonsHttpClient testCompile externalDependency.javaparser + + if (JavaVersion.current() >= JavaVersion.VERSION_11) { + //Custom dependency set is required for JDK11+ only source set + java11Implementation files(sourceSets.main.output.classesDirs) + java11Compile project(':data') + java11Compile project(':r2-core') + java11Compile project(':li-jersey-uri') + java11Compile project(':generator') + java11Compile project(':pegasus-common') + java11Compile project(':restli-common') + java11Compile project(':restli-client') + java11Compile project(':restli-server') + java11Compile externalDependency.caffeine + java11Compile externalDependency.commonsIo + java11Compile externalDependency.codemodel + java11Compile externalDependency.commonsCli + java11Compile externalDependency.commonsLang + java11Compile externalDependency.jacksonCore + java11Compile externalDependency.jacksonDataBind + java11Compile externalDependency.velocity + + java11Compile externalDependency.mockito + java11Compile externalDependency.testng + java11Compile externalDependency.junit + java11Compile externalDependency.commonsHttpClient + java11Compile externalDependency.javaparser + } } apply from: "${buildScriptDirPath}/restModel.gradle" diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java index 5817e33b0e..c5049f72c4 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java @@ -21,38 +21,27 @@ import com.linkedin.restli.server.annotations.ActionParam; import com.linkedin.restli.server.annotations.QueryParam; -import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Method; -import java.nio.file.Files; -import java.nio.file.FileVisitResult; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; -import com.sun.source.doctree.DocTree; -import com.sun.source.doctree.ReturnTree; -import com.sun.source.doctree.UnknownBlockTagTree; +import com.sun.javadoc.AnnotationDesc; +import com.sun.javadoc.ClassDoc; +import com.sun.javadoc.Doc; +import com.sun.javadoc.MethodDoc; +import com.sun.javadoc.ParamTag; +import com.sun.javadoc.Parameter; +import com.sun.javadoc.Tag; import org.apache.commons.io.output.NullWriter; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.lang.model.element.AnnotationMirror; -import javax.lang.model.element.AnnotationValue; -import javax.lang.model.element.Element; -import javax.lang.model.element.ExecutableElement; -import javax.lang.model.element.TypeElement; -import javax.lang.model.element.VariableElement; - /** * Specialized {@link DocsProvider} whose documentation comes from the Javadoc Doclet {@link RestLiDoclet}. @@ -87,48 +76,11 @@ public Set supportedFileExtensions() return Collections.singleton(".java"); } - /** - * Recursively collect all Java file paths under the sourcePaths if packageNames is null or empty. Else, only - * collect the Java file paths whose package name starts with packageNames. - * - * @param sourcePaths source paths to be queried - * @param packageNames target package names to be matched - * @return list of Java file paths - */ - public static List collectSourceFiles(List sourcePaths, List packageNames) throws IOException { - List sourceFiles = new ArrayList<>(); - for (String sourcePath : sourcePaths) { - Path basePath = Paths.get(sourcePath); - if (!Files.exists(basePath)) { - continue; - } - Files.walkFileTree(basePath, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - if (file.toString().endsWith(".java")) { - if (packageNames == null || packageNames.isEmpty()) { - sourceFiles.add(file.toString()); - } else { - String packageName = basePath.relativize(file.getParent()).toString().replace('/', '.'); - for (String targetPackageName : packageNames) { - if (packageName.startsWith(targetPackageName)) { - sourceFiles.add(file.toString()); - break; - } - } - } - } - return FileVisitResult.CONTINUE; - } - }); - } - return sourceFiles; - } - @Override public void registerSourceFiles(Collection sourceFileNames) { log.debug("Executing Javadoc tool..."); + final String flatClasspath; if (_classpath == null) { @@ -141,78 +93,89 @@ public void registerSourceFiles(Collection sourceFileNames) final PrintWriter sysoutWriter = new PrintWriter(System.out, true); final PrintWriter nullWriter = new PrintWriter(new NullWriter()); - - - List sourceFiles; - try + final List javadocArgs = new ArrayList<>(Arrays.asList("-classpath", + flatClasspath, + "-sourcepath", + StringUtils.join(_sourcePaths, ":"))); + if (_resourcePackages != null) { - sourceFiles = collectSourceFiles(Arrays.asList(_sourcePaths), - _resourcePackages == null ? null : Arrays.asList(_resourcePackages)); + javadocArgs.add("-subpackages"); + javadocArgs.add(StringUtils.join(_resourcePackages, ":")); } - catch (IOException e) + else { - throw new RuntimeException("Failed to collect source files", e); + javadocArgs.addAll(sourceFileNames); } _doclet = RestLiDoclet.generateDoclet(_apiName, sysoutWriter, nullWriter, nullWriter, - flatClasspath, - sourceFiles - ); + javadocArgs.toArray(new String[0])); } @Override public String getClassDoc(Class resourceClass) { - final TypeElement doc = _doclet.getClassDoc(resourceClass); + final ClassDoc doc = _doclet.getClassDoc(resourceClass); if (doc == null) { return null; } - return buildDoc(_doclet.getDocCommentStrForElement(doc)); + + return buildDoc(doc.commentText()); } - public String getClassDeprecatedTag(Class resourceClass) { - TypeElement typeElement = _doclet.getClassDoc(resourceClass); - if (typeElement == null) { + @Override + public String getClassDeprecatedTag(Class resourceClass) + { + final ClassDoc doc = _doclet.getClassDoc(resourceClass); + if (doc == null) + { return null; } - return formatDeprecatedTags(typeElement); + + return formatDeprecatedTags(doc); } - private String formatDeprecatedTags(Element element) { - List deprecatedTags = _doclet.getDeprecatedTags(element); - if (!deprecatedTags.isEmpty()) { + private static String formatDeprecatedTags(Doc doc) + { + Tag[] deprecatedTags = doc.tags("deprecated"); + if(deprecatedTags.length > 0) + { StringBuilder deprecatedText = new StringBuilder(); - for (int i = 0; i < deprecatedTags.size(); i++) { - deprecatedText.append(deprecatedTags.get(i)); - if (i < deprecatedTags.size() - 1) { + for(int i = 0; i < deprecatedTags.length; i++) + { + deprecatedText.append(deprecatedTags[i].text()); + if(i < deprecatedTags.length - 1) + { deprecatedText.append(" "); } } return deprecatedText.toString(); - } else { + } + else + { return null; } } + @Override public String getMethodDoc(Method method) { - final ExecutableElement doc = _doclet.getMethodDoc(method); + final MethodDoc doc = _doclet.getMethodDoc(method); if (doc == null) { return null; } - return buildDoc(_doclet.getDocCommentStrForElement(doc)); + return buildDoc(doc.commentText()); } @Override public String getMethodDeprecatedTag(Method method) { - final ExecutableElement doc = _doclet.getMethodDoc(method); + final MethodDoc doc = _doclet.getMethodDoc(method); if (doc == null) { return null; @@ -221,28 +184,32 @@ public String getMethodDeprecatedTag(Method method) return formatDeprecatedTags(doc); } - @Override public String getParamDoc(Method method, String name) { - final ExecutableElement methodDoc = _doclet.getMethodDoc(method); + final MethodDoc methodDoc = _doclet.getMethodDoc(method); if (methodDoc == null) { return null; } - Map paramTags = _doclet.getParamTags(methodDoc); - for (VariableElement parameter : methodDoc.getParameters()) + + for (Parameter parameter : methodDoc.parameters()) { - for (AnnotationMirror annotationMirror : parameter.getAnnotationMirrors()) + for (AnnotationDesc annotationDesc : parameter.annotations()) { - if (isQueryParamAnnotation(annotationMirror) || isActionParamAnnotation(annotationMirror)) + if (annotationDesc.isSynthesized()) { - for (Map.Entry entry : annotationMirror.getElementValues().entrySet()) + continue; + } + + if (isQueryParamAnnotation(annotationDesc) || isActionParamAnnotation(annotationDesc)) + { + for (AnnotationDesc.ElementValuePair pair : annotationDesc.elementValues()) { - if ("value".equals(entry.getKey().getSimpleName().toString()) && name.equals(entry.getValue().getValue())) + if ("value".equals(pair.element().name()) && name.equals(pair.value().value())) { - return paramTags.get(parameter.getSimpleName().toString()); + return getParamTagDoc(methodDoc, parameter.name()); } } } @@ -252,25 +219,34 @@ public String getParamDoc(Method method, String name) return null; } + private static String getParamTagDoc(MethodDoc methodDoc, String name) + { + for (ParamTag tag : methodDoc.paramTags()) + { + if (name.equals(tag.parameterName())) + { + return buildDoc(tag.parameterComment()); + } + } + + return null; + } + @Override public String getReturnDoc(Method method) { - ExecutableElement methodElement = _doclet.getMethodDoc(method); - if (methodElement != null) { - for (DocTree docTree : _doclet.getDocCommentTreeForMethod(method).getBlockTags()) { - if (!docTree.toString().toLowerCase().startsWith("@return")) { - continue; - } - DocTree.Kind kind = docTree.getKind(); - if (kind == DocTree.Kind.RETURN) { - ReturnTree returnTree = (ReturnTree) docTree; - return buildDoc(returnTree.getDescription().toString()); - } else if (kind == DocTree.Kind.UNKNOWN_BLOCK_TAG) { - UnknownBlockTagTree unknownBlockTagTree = (UnknownBlockTagTree) docTree; - return buildDoc(unknownBlockTagTree.getContent().toString()); + final MethodDoc methodDoc = _doclet.getMethodDoc(method); + if (methodDoc != null) + { + for (Tag tag : methodDoc.tags()) + { + if(tag.name().toLowerCase().equals("@return")) + { + return buildDoc(tag.text()); } } } + return null; } @@ -278,18 +254,20 @@ private static String buildDoc(String docText) { if (docText != null && !docText.isEmpty()) { - return DocletHelper.processDocCommentStr(docText); + return docText; } + return null; } - private static boolean isQueryParamAnnotation(AnnotationMirror annotationMirror) + + private static boolean isQueryParamAnnotation(AnnotationDesc annotationDesc) { - return QueryParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); + return QueryParam.class.getCanonicalName().equals(annotationDesc.annotationType().qualifiedName()); } - private static boolean isActionParamAnnotation(AnnotationMirror annotationMirror) + private static boolean isActionParamAnnotation(AnnotationDesc annotationDesc) { - return ActionParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); + return ActionParam.class.getCanonicalName().equals(annotationDesc.annotationType().qualifiedName()); } } \ No newline at end of file diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java index a50824e26e..1ae6cdc753 100644 --- a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java +++ b/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/RestLiDoclet.java @@ -17,35 +17,21 @@ package com.linkedin.restli.tools.idlgen; -import com.sun.source.doctree.DocCommentTree; -import com.sun.source.doctree.DocTree; -import com.sun.source.doctree.ParamTree; -import com.sun.source.doctree.DeprecatedTree; -import jdk.javadoc.doclet.Doclet; -import jdk.javadoc.doclet.DocletEnvironment; -import jdk.javadoc.doclet.Reporter; +import com.sun.javadoc.ClassDoc; +import com.sun.javadoc.MethodDoc; +import com.sun.javadoc.Parameter; +import com.sun.javadoc.RootDoc; +import com.sun.javadoc.Type; +import com.sun.tools.javadoc.Main; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import javax.lang.model.element.Element; -import javax.lang.model.element.ExecutableElement; -import javax.lang.model.element.TypeElement; -import javax.lang.model.SourceVersion; -import javax.lang.model.element.VariableElement; -import javax.lang.model.type.TypeMirror; -import javax.tools.DocumentationTool; -import javax.tools.JavaFileObject; -import javax.tools.StandardJavaFileManager; -import javax.tools.ToolProvider; + import java.io.PrintWriter; import java.lang.reflect.Method; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; /** @@ -55,77 +41,45 @@ * cleanly integrate the output into the {@link RestLiResourceModelExporter} tool. Thus, we're just * dumping the docs into a static Map which can be accessed by {@link RestLiResourceModelExporter}. * - * This class supports multiple runs of Javadoc Doclet API {@link DocumentationTool}. + * This class supports multiple runs of Javadoc Doclet API {@link Main#execute(String[])}. * Each run will be assigned an unique "Doclet ID", returned by - * {@link #generateDoclet(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String, List)}. + * {@link #generateDoclet(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String[])}. * The Doclet ID should be subsequently used to initialize {@link DocletDocsProvider}. * * This class is thread-safe. However, #generateJavadoc() will be synchronized. * * @author dellamag + * @see Main#execute(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String, String[]) */ -public class RestLiDoclet implements Doclet +public class RestLiDoclet { private static RestLiDoclet _currentDocLet = null; + private final DocInfo _docInfo; - private final DocletEnvironment _docEnv; /** - * Generate Javadoc and return the generated RestLiDoclet instance. + * Generate Javadoc and return associated Doclet ID. * This method is synchronized. * * @param programName Name of the program (for error messages). * @param errWriter PrintWriter to receive error messages. * @param warnWriter PrintWriter to receive warning messages. * @param noticeWriter PrintWriter to receive notice messages. - * @param flatClassPath Flat path to classes to be used. - * @param sourceFiles List of Java source files to be analyzed. - * @return the generated RestLiDoclet instance. + * @param args The command line parameters. + * @return an unique doclet ID which represent the subsequent Main#execute() run. + * @throws IllegalStateException if the generated doclet ID is already used. Try again. * @throws IllegalArgumentException if Javadoc fails to generate docs. */ public static synchronized RestLiDoclet generateDoclet(String programName, PrintWriter errWriter, PrintWriter warnWriter, PrintWriter noticeWriter, - String flatClassPath, - List sourceFiles - ) + String[] args) { - noticeWriter.println("Generating Javadoc for " + programName); - - DocumentationTool docTool = ToolProvider.getSystemDocumentationTool(); - StandardJavaFileManager fileManager = docTool.getStandardFileManager(null, null, null); - Iterable fileObjects = fileManager.getJavaFileObjectsFromPaths( - sourceFiles.stream().map(Paths::get).collect(Collectors.toList())); - - // Set up the Javadoc task options - List taskOptions = new ArrayList<>(); - taskOptions.add("-classpath"); - taskOptions.add(flatClassPath); - - // Create and run the Javadoc task - DocumentationTool.DocumentationTask task = docTool.getTask(errWriter, - fileManager, diagnostic -> { - switch (diagnostic.getKind()) { - case ERROR: - errWriter.println(diagnostic.getMessage(Locale.getDefault())); - break; - case WARNING: - warnWriter.println(diagnostic.getMessage(Locale.getDefault())); - break; - case NOTE: - noticeWriter.println(diagnostic.getMessage(Locale.getDefault())); - break; - } - }, - RestLiDoclet.class, - taskOptions, - fileObjects); - - boolean success = task.call(); - if (!success) + final int javadocRetCode = Main.execute(programName, errWriter, warnWriter, noticeWriter, RestLiDoclet.class.getName(), args); + if (javadocRetCode != 0) { - throw new IllegalArgumentException("Javadoc generation failed"); + throw new IllegalArgumentException("Javadoc failed with return code " + javadocRetCode); } return _currentDocLet; @@ -134,112 +88,80 @@ public static synchronized RestLiDoclet generateDoclet(String programName, /** * Entry point for Javadoc Doclet. * - * @param docEnv {@link DocletEnvironment} passed in by Javadoc + * @param root {@link RootDoc} passed in by Javadoc * @return is successful or not */ - @Override - public boolean run(DocletEnvironment docEnv) { + public static boolean start(RootDoc root) + { final DocInfo docInfo = new DocInfo(); - // Iterate through the TypeElements (class and interface declarations) - for (Element element : docEnv.getIncludedElements()) { - if (element instanceof TypeElement) { - TypeElement typeElement = (TypeElement) element; - docInfo.setClassDoc(typeElement.getQualifiedName().toString(), typeElement); - - // Iterate through the methods of the TypeElement - for (Element enclosedElement : typeElement.getEnclosedElements()) { - if (enclosedElement instanceof ExecutableElement) { - ExecutableElement methodElement = (ExecutableElement) enclosedElement; - docInfo.setMethodDoc(MethodIdentity.create(methodElement), methodElement); - } - } + for (ClassDoc classDoc : root.classes()) + { + docInfo.setClassDoc(classDoc.qualifiedName(), classDoc); + + for (MethodDoc methodDoc : classDoc.methods()) + { + docInfo.setMethodDoc(MethodIdentity.create(methodDoc), methodDoc); } } - _currentDocLet = new RestLiDoclet(docInfo, docEnv); + _currentDocLet = new RestLiDoclet(docInfo); return true; } - @Override - public void init(Locale locale, Reporter reporter) { - // no-ops - } - - @Override - public String getName() { - return this.getClass().getSimpleName(); - } - - @Override - public Set getSupportedOptions() { - return Set.of(); - } - - @Override - public SourceVersion getSupportedSourceVersion() { - return SourceVersion.latest(); - } - - private RestLiDoclet(DocInfo docInfo, DocletEnvironment docEnv) + private RestLiDoclet(DocInfo docInfo) { _docInfo = docInfo; - _docEnv = docEnv; } /** - * The reason why we create a public empty constructor is because JavadocTaskImpl in JDK 11 requires it when using reflection. - * Otherwise, there will be NoSuchMethodException: com.linkedin.restli.tools.idlgen.RestLiDoclet.() - */ - public RestLiDoclet() { - _docInfo = null; - _docEnv = null; - } - - /** - * Query Javadoc {@link TypeElement} for the specified resource class. + * Query Javadoc {@link ClassDoc} for the specified resource class. * * @param resourceClass resource class to be queried - * @return corresponding {@link TypeElement} + * @return corresponding {@link ClassDoc} */ - public TypeElement getClassDoc(Class resourceClass) + public ClassDoc getClassDoc(Class resourceClass) { return _docInfo.getClassDoc(resourceClass.getCanonicalName()); } /** - * Query Javadoc {@link ExecutableElement} for the specified Java method. + * Query Javadoc {@link MethodDoc} for the specified Java method. * * @param method Java method to be queried - * @return corresponding {@link ExecutableElement} + * @return corresponding {@link MethodDoc} */ - public ExecutableElement getMethodDoc(Method method) + public MethodDoc getMethodDoc(Method method) { final MethodIdentity methodId = MethodIdentity.create(method); - return _docInfo.getMethodDoc(methodId); + return _docInfo.getMethodDoc(methodId); } private static class DocInfo { - public TypeElement getClassDoc(String className) { + public ClassDoc getClassDoc(String className) + { return _classNameToClassDoc.get(className); } - public ExecutableElement getMethodDoc(MethodIdentity methodId) { + public MethodDoc getMethodDoc(MethodIdentity methodId) + { return _methodIdToMethodDoc.get(methodId); } - public void setClassDoc(String className, TypeElement classDoc) { + public void setClassDoc(String className, ClassDoc classDoc) + { _classNameToClassDoc.put(className, classDoc); } - public void setMethodDoc(MethodIdentity methodId, ExecutableElement methodDoc) { + public void setMethodDoc(MethodIdentity methodId, MethodDoc methodDoc) + { _methodIdToMethodDoc.put(methodId, methodDoc); } - private final Map _classNameToClassDoc = new HashMap<>(); - private final Map _methodIdToMethodDoc = new HashMap<>(); + private final Map _classNameToClassDoc = new HashMap<>(); + private final Map _methodIdToMethodDoc = new HashMap<>(); } private static class MethodIdentity @@ -260,16 +182,16 @@ public static MethodIdentity create(Method method) return new MethodIdentity(method.getDeclaringClass().getName() + "." + method.getName(), parameterTypeNames); } - public static MethodIdentity create(ExecutableElement method) + public static MethodIdentity create(MethodDoc method) { final List parameterTypeNames = new ArrayList<>(); - for (VariableElement param : method.getParameters()) { - TypeMirror type = param.asType(); - parameterTypeNames.add(DocletHelper.getCanonicalName(type.toString())); + for (Parameter param: method.parameters()) + { + Type type = param.type(); + parameterTypeNames.add(type.qualifiedTypeName() + type.dimension()); } - return new MethodIdentity(method.getEnclosingElement().toString() + "." + method.getSimpleName().toString(), - parameterTypeNames); + return new MethodIdentity(method.qualifiedName(), parameterTypeNames); } private MethodIdentity(String methodQualifiedName, List parameterTypeNames) @@ -315,91 +237,4 @@ public boolean equals(Object obj) private final String _methodQualifiedName; private final List _parameterTypeNames; } - - /** - * Get the list of deprecated tags for the specified element. - * - * @param element {@link Element} to be queried - * @return list of deprecated tags for the specified element - */ - public List getDeprecatedTags(Element element) { - List deprecatedTags = new ArrayList<>(); - DocCommentTree docCommentTree = getDocCommentTreeForElement(element); - if (docCommentTree == null) { - return deprecatedTags; - } - for (DocTree docTree :docCommentTree.getBlockTags()) { - if (docTree.getKind() == DocTree.Kind.DEPRECATED) { - DeprecatedTree deprecatedTree = (DeprecatedTree) docTree; - String deprecatedComment = deprecatedTree.getBody().toString(); - deprecatedTags.add(deprecatedComment); - } - } - return deprecatedTags; - } - - /** - * Get the map from param name to param comment for the specified executableElement. - * - * @param executableElement {@link ExecutableElement} to be queried - * @return map from param name to param comment for the specified executableElement - */ - public Map getParamTags(ExecutableElement executableElement) { - Map paramTags = new HashMap<>(); - DocCommentTree docCommentTree = getDocCommentTreeForElement(executableElement); - if (docCommentTree == null) { - return paramTags; - } - for (DocTree docTree : docCommentTree.getBlockTags()) { - if (docTree.getKind() == DocTree.Kind.PARAM) { - ParamTree paramTree = (ParamTree) docTree; - String paramName = paramTree.getName().toString(); - String paramComment = paramTree.getDescription().toString(); - if (paramComment != null) { - paramTags.put(paramName, paramComment); - } - } - } - return paramTags; - } - - /** - * Get the {@link DocCommentTree} for the specified element. - * - * @param element {@link Element} to be queried - * @return {@link DocCommentTree} for the specified element - */ - public DocCommentTree getDocCommentTreeForElement(Element element) { - return element == null ? null : _docEnv.getDocTrees().getDocCommentTree(element); - } - - /** - * Get the Doc Comment string for the specified element. - * - * @param element {@link Element} to be queried - * @return Doc Comment string for the specified element - */ - public String getDocCommentStrForElement(Element element) { - DocCommentTree docCommentTree = getDocCommentTreeForElement(element); - return docCommentTree == null ? null : docCommentTree.getFullBody().toString(); - } - - /** - * Get the {@link DocCommentTree} for the specified method. - * - * @param method {@link Method} to be queried - * @return {@link DocCommentTree} for the specified method - */ - public DocCommentTree getDocCommentTreeForMethod(Method method) { - TypeElement typeElement = getClassDoc(method.getDeclaringClass()); - if (typeElement == null) { - return null; - } - for (Element element : typeElement.getEnclosedElements()) { - if (element.getSimpleName().toString().equals(method.getName())) { - return getDocCommentTreeForElement(element); - } - } - return null; - } } \ No newline at end of file diff --git a/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java new file mode 100644 index 0000000000..5817e33b0e --- /dev/null +++ b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletDocsProvider.java @@ -0,0 +1,295 @@ +/* + Copyright (c) 2012 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.restli.tools.idlgen; + + +import com.linkedin.restli.internal.server.model.ResourceModelEncoder.DocsProvider; +import com.linkedin.restli.server.annotations.ActionParam; +import com.linkedin.restli.server.annotations.QueryParam; + +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.FileVisitResult; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.sun.source.doctree.DocTree; +import com.sun.source.doctree.ReturnTree; +import com.sun.source.doctree.UnknownBlockTagTree; +import org.apache.commons.io.output.NullWriter; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.lang.model.element.AnnotationMirror; +import javax.lang.model.element.AnnotationValue; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.element.VariableElement; + + +/** + * Specialized {@link DocsProvider} whose documentation comes from the Javadoc Doclet {@link RestLiDoclet}. + * + * @author dellamag + */ +public class DocletDocsProvider implements DocsProvider +{ + private static final Logger log = LoggerFactory.getLogger(DocletDocsProvider.class); + + private final String _apiName; + private final String[] _classpath; + private final String[] _sourcePaths; + private final String[] _resourcePackages; + + private RestLiDoclet _doclet; + + public DocletDocsProvider(String apiName, + String[] classpath, + String[] sourcePaths, + String[] resourcePackages) + { + _apiName = apiName; + _classpath = classpath; + _sourcePaths = sourcePaths; + _resourcePackages = resourcePackages; + } + + @Override + public Set supportedFileExtensions() + { + return Collections.singleton(".java"); + } + + /** + * Recursively collect all Java file paths under the sourcePaths if packageNames is null or empty. Else, only + * collect the Java file paths whose package name starts with packageNames. + * + * @param sourcePaths source paths to be queried + * @param packageNames target package names to be matched + * @return list of Java file paths + */ + public static List collectSourceFiles(List sourcePaths, List packageNames) throws IOException { + List sourceFiles = new ArrayList<>(); + for (String sourcePath : sourcePaths) { + Path basePath = Paths.get(sourcePath); + if (!Files.exists(basePath)) { + continue; + } + Files.walkFileTree(basePath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().endsWith(".java")) { + if (packageNames == null || packageNames.isEmpty()) { + sourceFiles.add(file.toString()); + } else { + String packageName = basePath.relativize(file.getParent()).toString().replace('/', '.'); + for (String targetPackageName : packageNames) { + if (packageName.startsWith(targetPackageName)) { + sourceFiles.add(file.toString()); + break; + } + } + } + } + return FileVisitResult.CONTINUE; + } + }); + } + return sourceFiles; + } + + @Override + public void registerSourceFiles(Collection sourceFileNames) + { + log.debug("Executing Javadoc tool..."); + final String flatClasspath; + if (_classpath == null) + { + flatClasspath = System.getProperty("java.class.path"); + } + else + { + flatClasspath = StringUtils.join(_classpath, ":"); + } + + final PrintWriter sysoutWriter = new PrintWriter(System.out, true); + final PrintWriter nullWriter = new PrintWriter(new NullWriter()); + + + List sourceFiles; + try + { + sourceFiles = collectSourceFiles(Arrays.asList(_sourcePaths), + _resourcePackages == null ? null : Arrays.asList(_resourcePackages)); + } + catch (IOException e) + { + throw new RuntimeException("Failed to collect source files", e); + } + + _doclet = RestLiDoclet.generateDoclet(_apiName, + sysoutWriter, + nullWriter, + nullWriter, + flatClasspath, + sourceFiles + ); + } + + @Override + public String getClassDoc(Class resourceClass) + { + final TypeElement doc = _doclet.getClassDoc(resourceClass); + if (doc == null) + { + return null; + } + return buildDoc(_doclet.getDocCommentStrForElement(doc)); + } + + public String getClassDeprecatedTag(Class resourceClass) { + TypeElement typeElement = _doclet.getClassDoc(resourceClass); + if (typeElement == null) { + return null; + } + return formatDeprecatedTags(typeElement); + } + + private String formatDeprecatedTags(Element element) { + List deprecatedTags = _doclet.getDeprecatedTags(element); + if (!deprecatedTags.isEmpty()) { + StringBuilder deprecatedText = new StringBuilder(); + for (int i = 0; i < deprecatedTags.size(); i++) { + deprecatedText.append(deprecatedTags.get(i)); + if (i < deprecatedTags.size() - 1) { + deprecatedText.append(" "); + } + } + return deprecatedText.toString(); + } else { + return null; + } + } + @Override + public String getMethodDoc(Method method) + { + final ExecutableElement doc = _doclet.getMethodDoc(method); + if (doc == null) + { + return null; + } + + return buildDoc(_doclet.getDocCommentStrForElement(doc)); + } + + @Override + public String getMethodDeprecatedTag(Method method) + { + final ExecutableElement doc = _doclet.getMethodDoc(method); + if (doc == null) + { + return null; + } + + return formatDeprecatedTags(doc); + } + + + @Override + public String getParamDoc(Method method, String name) + { + final ExecutableElement methodDoc = _doclet.getMethodDoc(method); + + if (methodDoc == null) + { + return null; + } + Map paramTags = _doclet.getParamTags(methodDoc); + for (VariableElement parameter : methodDoc.getParameters()) + { + for (AnnotationMirror annotationMirror : parameter.getAnnotationMirrors()) + { + if (isQueryParamAnnotation(annotationMirror) || isActionParamAnnotation(annotationMirror)) + { + for (Map.Entry entry : annotationMirror.getElementValues().entrySet()) + { + if ("value".equals(entry.getKey().getSimpleName().toString()) && name.equals(entry.getValue().getValue())) + { + return paramTags.get(parameter.getSimpleName().toString()); + } + } + } + } + } + + return null; + } + + @Override + public String getReturnDoc(Method method) + { + ExecutableElement methodElement = _doclet.getMethodDoc(method); + if (methodElement != null) { + for (DocTree docTree : _doclet.getDocCommentTreeForMethod(method).getBlockTags()) { + if (!docTree.toString().toLowerCase().startsWith("@return")) { + continue; + } + DocTree.Kind kind = docTree.getKind(); + if (kind == DocTree.Kind.RETURN) { + ReturnTree returnTree = (ReturnTree) docTree; + return buildDoc(returnTree.getDescription().toString()); + } else if (kind == DocTree.Kind.UNKNOWN_BLOCK_TAG) { + UnknownBlockTagTree unknownBlockTagTree = (UnknownBlockTagTree) docTree; + return buildDoc(unknownBlockTagTree.getContent().toString()); + } + } + } + return null; + } + + private static String buildDoc(String docText) + { + if (docText != null && !docText.isEmpty()) + { + return DocletHelper.processDocCommentStr(docText); + } + return null; + } + + private static boolean isQueryParamAnnotation(AnnotationMirror annotationMirror) + { + return QueryParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); + } + + private static boolean isActionParamAnnotation(AnnotationMirror annotationMirror) + { + return ActionParam.class.getCanonicalName().equals(annotationMirror.getAnnotationType().toString()); + } +} \ No newline at end of file diff --git a/restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletHelper.java b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletHelper.java similarity index 100% rename from restli-tools/src/main/java/com/linkedin/restli/tools/idlgen/DocletHelper.java rename to restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/DocletHelper.java diff --git a/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/RestLiDoclet.java b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/RestLiDoclet.java new file mode 100644 index 0000000000..a50824e26e --- /dev/null +++ b/restli-tools/src/main/java11/com/linkedin/restli/tools/idlgen/RestLiDoclet.java @@ -0,0 +1,405 @@ +/* + Copyright (c) 2012 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.restli.tools.idlgen; + + +import com.sun.source.doctree.DocCommentTree; +import com.sun.source.doctree.DocTree; +import com.sun.source.doctree.ParamTree; +import com.sun.source.doctree.DeprecatedTree; +import jdk.javadoc.doclet.Doclet; +import jdk.javadoc.doclet.DocletEnvironment; +import jdk.javadoc.doclet.Reporter; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.VariableElement; +import javax.lang.model.type.TypeMirror; +import javax.tools.DocumentationTool; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; +import java.io.PrintWriter; +import java.lang.reflect.Method; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * Custom Javadoc processor that merges documentation into the restspec.json. The embedded Javadoc + * generator is basically a commandline tool wrapper and it runs in complete isolation from the rest + * of the application. Due to the fact that the Javadoc tool instantiates RestLiDoclet, we cannot + * cleanly integrate the output into the {@link RestLiResourceModelExporter} tool. Thus, we're just + * dumping the docs into a static Map which can be accessed by {@link RestLiResourceModelExporter}. + * + * This class supports multiple runs of Javadoc Doclet API {@link DocumentationTool}. + * Each run will be assigned an unique "Doclet ID", returned by + * {@link #generateDoclet(String, java.io.PrintWriter, java.io.PrintWriter, java.io.PrintWriter, String, List)}. + * The Doclet ID should be subsequently used to initialize {@link DocletDocsProvider}. + * + * This class is thread-safe. However, #generateJavadoc() will be synchronized. + * + * @author dellamag + */ +public class RestLiDoclet implements Doclet +{ + private static RestLiDoclet _currentDocLet = null; + private final DocInfo _docInfo; + private final DocletEnvironment _docEnv; + + /** + * Generate Javadoc and return the generated RestLiDoclet instance. + * This method is synchronized. + * + * @param programName Name of the program (for error messages). + * @param errWriter PrintWriter to receive error messages. + * @param warnWriter PrintWriter to receive warning messages. + * @param noticeWriter PrintWriter to receive notice messages. + * @param flatClassPath Flat path to classes to be used. + * @param sourceFiles List of Java source files to be analyzed. + * @return the generated RestLiDoclet instance. + * @throws IllegalArgumentException if Javadoc fails to generate docs. + */ + public static synchronized RestLiDoclet generateDoclet(String programName, + PrintWriter errWriter, + PrintWriter warnWriter, + PrintWriter noticeWriter, + String flatClassPath, + List sourceFiles + ) + { + noticeWriter.println("Generating Javadoc for " + programName); + + DocumentationTool docTool = ToolProvider.getSystemDocumentationTool(); + StandardJavaFileManager fileManager = docTool.getStandardFileManager(null, null, null); + Iterable fileObjects = fileManager.getJavaFileObjectsFromPaths( + sourceFiles.stream().map(Paths::get).collect(Collectors.toList())); + + // Set up the Javadoc task options + List taskOptions = new ArrayList<>(); + taskOptions.add("-classpath"); + taskOptions.add(flatClassPath); + + // Create and run the Javadoc task + DocumentationTool.DocumentationTask task = docTool.getTask(errWriter, + fileManager, diagnostic -> { + switch (diagnostic.getKind()) { + case ERROR: + errWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case WARNING: + warnWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + case NOTE: + noticeWriter.println(diagnostic.getMessage(Locale.getDefault())); + break; + } + }, + RestLiDoclet.class, + taskOptions, + fileObjects); + + boolean success = task.call(); + if (!success) + { + throw new IllegalArgumentException("Javadoc generation failed"); + } + + return _currentDocLet; + } + + /** + * Entry point for Javadoc Doclet. + * + * @param docEnv {@link DocletEnvironment} passed in by Javadoc + * @return is successful or not + */ + @Override + public boolean run(DocletEnvironment docEnv) { + final DocInfo docInfo = new DocInfo(); + + // Iterate through the TypeElements (class and interface declarations) + for (Element element : docEnv.getIncludedElements()) { + if (element instanceof TypeElement) { + TypeElement typeElement = (TypeElement) element; + docInfo.setClassDoc(typeElement.getQualifiedName().toString(), typeElement); + + // Iterate through the methods of the TypeElement + for (Element enclosedElement : typeElement.getEnclosedElements()) { + if (enclosedElement instanceof ExecutableElement) { + ExecutableElement methodElement = (ExecutableElement) enclosedElement; + docInfo.setMethodDoc(MethodIdentity.create(methodElement), methodElement); + } + } + } + } + + _currentDocLet = new RestLiDoclet(docInfo, docEnv); + + return true; + } + + @Override + public void init(Locale locale, Reporter reporter) { + // no-ops + } + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + + @Override + public Set getSupportedOptions() { + return Set.of(); + } + + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latest(); + } + + private RestLiDoclet(DocInfo docInfo, DocletEnvironment docEnv) + { + _docInfo = docInfo; + _docEnv = docEnv; + } + + /** + * The reason why we create a public empty constructor is because JavadocTaskImpl in JDK 11 requires it when using reflection. + * Otherwise, there will be NoSuchMethodException: com.linkedin.restli.tools.idlgen.RestLiDoclet.() + */ + public RestLiDoclet() { + _docInfo = null; + _docEnv = null; + } + + /** + * Query Javadoc {@link TypeElement} for the specified resource class. + * + * @param resourceClass resource class to be queried + * @return corresponding {@link TypeElement} + */ + public TypeElement getClassDoc(Class resourceClass) + { + return _docInfo.getClassDoc(resourceClass.getCanonicalName()); + } + + /** + * Query Javadoc {@link ExecutableElement} for the specified Java method. + * + * @param method Java method to be queried + * @return corresponding {@link ExecutableElement} + */ + public ExecutableElement getMethodDoc(Method method) + { + final MethodIdentity methodId = MethodIdentity.create(method); + return _docInfo.getMethodDoc(methodId); + } + + private static class DocInfo + { + public TypeElement getClassDoc(String className) { + return _classNameToClassDoc.get(className); + } + + public ExecutableElement getMethodDoc(MethodIdentity methodId) { + return _methodIdToMethodDoc.get(methodId); + } + + public void setClassDoc(String className, TypeElement classDoc) { + _classNameToClassDoc.put(className, classDoc); + } + + public void setMethodDoc(MethodIdentity methodId, ExecutableElement methodDoc) { + _methodIdToMethodDoc.put(methodId, methodDoc); + } + + private final Map _classNameToClassDoc = new HashMap<>(); + private final Map _methodIdToMethodDoc = new HashMap<>(); + } + + private static class MethodIdentity + { + public static MethodIdentity create(Method method) + { + final List parameterTypeNames = new ArrayList<>(); + + // type parameters are not included in identity because of differences between reflection and Doclet: + // e.g. for Collection: + // reflection Type.toString() -> Collection + // Doclet Type.toString() -> Collection + for (Class paramClass: method.getParameterTypes()) + { + parameterTypeNames.add(paramClass.getCanonicalName()); + } + + return new MethodIdentity(method.getDeclaringClass().getName() + "." + method.getName(), parameterTypeNames); + } + + public static MethodIdentity create(ExecutableElement method) + { + final List parameterTypeNames = new ArrayList<>(); + for (VariableElement param : method.getParameters()) { + TypeMirror type = param.asType(); + parameterTypeNames.add(DocletHelper.getCanonicalName(type.toString())); + } + + return new MethodIdentity(method.getEnclosingElement().toString() + "." + method.getSimpleName().toString(), + parameterTypeNames); + } + + private MethodIdentity(String methodQualifiedName, List parameterTypeNames) + { + _methodQualifiedName = methodQualifiedName; + _parameterTypeNames = parameterTypeNames; + } + + @Override + public int hashCode() + { + return new HashCodeBuilder(17, 29). + append(_methodQualifiedName). + append(_parameterTypeNames). + toHashCode(); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + { + return true; + } + + if (obj == null) + { + return false; + } + + if (getClass() != obj.getClass()) + { + return false; + } + + final MethodIdentity other = (MethodIdentity) obj; + return new EqualsBuilder(). + append(_methodQualifiedName, other._methodQualifiedName). + append(_parameterTypeNames, other._parameterTypeNames). + isEquals(); + } + + private final String _methodQualifiedName; + private final List _parameterTypeNames; + } + + /** + * Get the list of deprecated tags for the specified element. + * + * @param element {@link Element} to be queried + * @return list of deprecated tags for the specified element + */ + public List getDeprecatedTags(Element element) { + List deprecatedTags = new ArrayList<>(); + DocCommentTree docCommentTree = getDocCommentTreeForElement(element); + if (docCommentTree == null) { + return deprecatedTags; + } + for (DocTree docTree :docCommentTree.getBlockTags()) { + if (docTree.getKind() == DocTree.Kind.DEPRECATED) { + DeprecatedTree deprecatedTree = (DeprecatedTree) docTree; + String deprecatedComment = deprecatedTree.getBody().toString(); + deprecatedTags.add(deprecatedComment); + } + } + return deprecatedTags; + } + + /** + * Get the map from param name to param comment for the specified executableElement. + * + * @param executableElement {@link ExecutableElement} to be queried + * @return map from param name to param comment for the specified executableElement + */ + public Map getParamTags(ExecutableElement executableElement) { + Map paramTags = new HashMap<>(); + DocCommentTree docCommentTree = getDocCommentTreeForElement(executableElement); + if (docCommentTree == null) { + return paramTags; + } + for (DocTree docTree : docCommentTree.getBlockTags()) { + if (docTree.getKind() == DocTree.Kind.PARAM) { + ParamTree paramTree = (ParamTree) docTree; + String paramName = paramTree.getName().toString(); + String paramComment = paramTree.getDescription().toString(); + if (paramComment != null) { + paramTags.put(paramName, paramComment); + } + } + } + return paramTags; + } + + /** + * Get the {@link DocCommentTree} for the specified element. + * + * @param element {@link Element} to be queried + * @return {@link DocCommentTree} for the specified element + */ + public DocCommentTree getDocCommentTreeForElement(Element element) { + return element == null ? null : _docEnv.getDocTrees().getDocCommentTree(element); + } + + /** + * Get the Doc Comment string for the specified element. + * + * @param element {@link Element} to be queried + * @return Doc Comment string for the specified element + */ + public String getDocCommentStrForElement(Element element) { + DocCommentTree docCommentTree = getDocCommentTreeForElement(element); + return docCommentTree == null ? null : docCommentTree.getFullBody().toString(); + } + + /** + * Get the {@link DocCommentTree} for the specified method. + * + * @param method {@link Method} to be queried + * @return {@link DocCommentTree} for the specified method + */ + public DocCommentTree getDocCommentTreeForMethod(Method method) { + TypeElement typeElement = getClassDoc(method.getDeclaringClass()); + if (typeElement == null) { + return null; + } + for (Element element : typeElement.getEnclosedElements()) { + if (element.getSimpleName().toString().equals(method.getName())) { + return getDocCommentTreeForElement(element); + } + } + return null; + } +} \ No newline at end of file diff --git a/restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java b/restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java deleted file mode 100644 index 9ec837a348..0000000000 --- a/restli-tools/src/test/java/com/linkedin/restli/tools/idlgen/TestDocletHelper.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - Copyright (c) 2012 LinkedIn Corp. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package com.linkedin.restli.tools.idlgen; - - -import org.testng.Assert; -import org.testng.annotations.Test; - - -/** - * Tests to ensure that util methods in {@link DocletHelper} work correctly. - * - * @author Yan Zhou - */ -public class TestDocletHelper -{ - @Test - public void testGetCanonicalName() { - // input is null, so no op - String input = null; - String actualOutput = DocletHelper.getCanonicalName(input); - String expectedOutput = null; - Assert.assertTrue(actualOutput == expectedOutput); - - // input does not match the pattern, so no op - input = "asdfljk sdf \n * "; - actualOutput = DocletHelper.getCanonicalName(input); - expectedOutput = input; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches the pattern i.e. <*>, so relevant substring is removed - input = "))?? )* * "; - actualOutput = DocletHelper.getCanonicalName(input); - expectedOutput = "))?? )* * "; - Assert.assertEquals(actualOutput, expectedOutput); - } - - @Test - public void testProcessDocCommentStr() { - // input is null, so no op - String input = null; - String actualOutput = DocletHelper.processDocCommentStr(input); - String expectedOutput = null; - Assert.assertTrue(actualOutput == expectedOutput); - - // input does not match the pattern, so no op - input = "asdf \n () , ? * "; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = input; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches pattern, i.e. ,{@xxx }, so redundant commas are removed - input = " ,{@xxx }, "; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = " {@xxx } "; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches the pattern, i.e. ,{@xxx } so redundant commas are removed - input = " ,{@xxx } "; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = " {@xxx } "; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches, i.e. ,>, so redundant commas are removed - input = " ? ,>, ,"; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = " ? > ,"; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches the pattern, i.e. ,<, so redundant commas are removed - input = " ? ,<, ,"; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = " ? < ,"; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches the pattern, i.e. ,< so redundant commas are removed - input = " ? ,< ,"; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = " ? < ,"; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches the pattern, i.e. ,<,>, so redundant commas are removed - input = " ,<,>, ? *(,"; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = " <> ? *(,"; - Assert.assertEquals(actualOutput, expectedOutput); - - // input matches the pattern, i.e. ,, so redundant commas are removed - input = " ,,,?"; - actualOutput = DocletHelper.processDocCommentStr(input); - expectedOutput = " ,?"; - Assert.assertEquals(actualOutput, expectedOutput); - } -}