Skip to content

Commit 73bf620

Browse files
author
klessard_expedia
committed
Fixing javadoc generation scripts
1 parent e9c854f commit 73bf620

File tree

2 files changed

+223
-0
lines changed

2 files changed

+223
-0
lines changed

‎docs/tools/build_java_api_docs.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Lint as: python3
2+
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
"""Generate TensorFlow Java reference docs for TensorFlow.org."""
17+
from __future__ import absolute_import
18+
from __future__ import division
19+
from __future__ import print_function
20+
21+
import pathlib
22+
import shutil
23+
import tempfile
24+
import io
25+
import requests
26+
import zipfile
27+
from git import Repo
28+
29+
from absl import app
30+
from absl import flags
31+
32+
from tensorflow_docs.api_generator import gen_java
33+
34+
FLAGS = flags.FLAGS
35+
36+
NDARRAY_VERSION = 'v1.0.0'
37+
JAVACPP_VERSION = '1.5.11'
38+
PROTOBUF_VERSION = 'v3.21.9'
39+
40+
# __file__ is the path to this file
41+
TOOLS_DIR = pathlib.Path(__file__).resolve().parent
42+
DOCS_DIR = TOOLS_DIR.parent
43+
REPO_ROOT = DOCS_DIR.parent
44+
DOC_OUTPUT_DIR = DOCS_DIR.joinpath("output")
45+
46+
SECTION_LABELS = {
47+
'org.tensorflow': 'Core',
48+
'org.tensorflow.ndarray': 'NdArray',
49+
'org.tensorflow.framework': 'Framework',
50+
}
51+
52+
# These flags are required by infrastructure, not all of them are used.
53+
flags.DEFINE_string('output_dir', f"{DOC_OUTPUT_DIR}",
54+
("Use this branch as the root version and don't"
55+
' create in version directory'))
56+
57+
flags.DEFINE_string('site_path', 'api_docs/',
58+
'Path prefix in the _toc.yaml')
59+
60+
flags.DEFINE_string('code_url_prefix', None,
61+
'[UNUSED] The url prefix for links to code.')
62+
63+
flags.DEFINE_bool(
64+
'search_hints', True,
65+
'[UNUSED] Include metadata search hints in the generated files')
66+
67+
68+
def checkout_repo(repo_url: str, target_dir_name: str, version: str):
69+
local_repo_path = f"{REPO_ROOT}/{target_dir_name}"
70+
if not pathlib.Path(local_repo_path).exists():
71+
local_repo = Repo.clone_from(repo_url, local_repo_path)
72+
else:
73+
local_repo = Repo(local_repo_path)
74+
local_repo.remotes['origin'].fetch()
75+
local_repo.git.checkout(version)
76+
77+
78+
def overlay(from_root, to_root):
79+
for from_path in pathlib.Path(from_root).rglob('*'):
80+
relpath = from_path.relative_to(from_root)
81+
to_path = to_root/relpath
82+
if from_path.is_file():
83+
assert not to_path.exists()
84+
shutil.copyfile(from_path, to_path)
85+
else:
86+
to_path.mkdir(exist_ok=True)
87+
88+
89+
def main(unused_argv):
90+
checkout_repo('https://github.com/tensorflow/java-ndarray', 'ndarray', NDARRAY_VERSION)
91+
checkout_repo('https://github.com/bytedeco/javacpp', 'javacpp', JAVACPP_VERSION)
92+
response = requests.get('https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.21.9/protobuf-java-3.21.9-sources.jar')
93+
with zipfile.ZipFile(io.BytesIO(response.content)) as z:
94+
z.extractall(f"{REPO_ROOT}/protobuf")
95+
response = requests.get('https://repo1.maven.org/maven2/org/osgi/osgi.annotation/8.1.0/osgi.annotation-8.1.0-sources.jar')
96+
with zipfile.ZipFile(io.BytesIO(response.content)) as z:
97+
z.extractall(f"{REPO_ROOT}/osgi")
98+
99+
merged_source = pathlib.Path(tempfile.mkdtemp())
100+
(merged_source / 'java/org').mkdir(parents=True)
101+
shutil.copytree(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/', merged_source/'java/org/tensorflow')
102+
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow', merged_source/'java/org/tensorflow')
103+
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow', merged_source/'java/org/tensorflow')
104+
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/', merged_source/'java/org/tensorflow/')
105+
overlay(REPO_ROOT/'tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/', merged_source/'java/org/tensorflow/')
106+
shutil.copytree(REPO_ROOT/'tensorflow-framework/src/main/java/org/tensorflow/framework', merged_source/'java/org/tensorflow/framework')
107+
shutil.copytree(REPO_ROOT/'ndarray/ndarray/src/main/java/org/tensorflow/ndarray', merged_source/'java/org/tensorflow/ndarray')
108+
shutil.copytree(REPO_ROOT/'javacpp/src/main/java/org/bytedeco', merged_source/'java/org/bytedeco')
109+
shutil.copytree(REPO_ROOT/'protobuf/com/', merged_source/'java/com')
110+
shutil.copytree(REPO_ROOT/'osgi/org/osgi', merged_source/'java/org/osgi')
111+
112+
gen_java.gen_java_docs(
113+
package='org.tensorflow',
114+
source_path=merged_source / 'java',
115+
output_dir=pathlib.Path(FLAGS.output_dir),
116+
site_path=pathlib.Path(FLAGS.site_path),
117+
section_labels=SECTION_LABELS,
118+
# Uncomment for local testing:
119+
script_path=pathlib.Path(TOOLS_DIR, 'run-javadoc-for-tf-local.sh'),
120+
)
121+
122+
123+
if __name__ == '__main__':
124+
flags.mark_flags_as_required(['output_dir'])
125+
app.run(main)

‎docs/tools/run-javadoc-for-tf-local.sh‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
export JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home # Or change to any JDK 11 home path
5+
6+
# https://android.googlesource.com/platform/external/doclava/
7+
# There's a debian package:
8+
# https://packages.debian.org/unstable/doclava-aosp
9+
# Install with:
10+
#
11+
# $ sudo apt install doclava-aosp #v 6.0.1+r55-1+build1
12+
#
13+
# https://unix.stackexchange.com/questions/594841/how-do-i-assign-a-value-to-a-bash-variable-iff-that-variable-is-null-unassigned
14+
DOCLAVA_JAR=${DOCLAVA_JAR:-'tools/lib/doclava.jar'} # Build lib locally
15+
16+
17+
# Install java clear silver templates with:
18+
#
19+
# $ sudo apt install libjsilver-aosp-java #v 6.0.1+r55-1+build1
20+
JSILVER_JAR=${JSILVER_JAR:-'tools/lib/jsilver.jar'} # Build lib locally
21+
22+
23+
######### DELETE OUTPUT_DIR #################
24+
25+
# Empty the output directory in case a class has been deleted
26+
rm -rf "${OUTPUT_DIR:?}"/*
27+
############ RUN DOCLAVA ###################
28+
29+
# $FEDERATED_DOCS is a space-separated string of url,file pairs.
30+
read -a api_pairs <<< "${FEDERATED_DOCS}"
31+
FEDERATED_PARAMS=""
32+
for i in "${!api_pairs[@]}"; do
33+
api_pair_str="${api_pairs[$i]}" # "url,api.txt"
34+
read -a api_pair <<< "${api_pair_str//,/ }"
35+
# Using the index as the API "name", build the federation params. Note that
36+
# using 0 as an API name will evaluate to false and cause rendering bugs,
37+
# so we preface with "api_".
38+
FEDERATED_PARAMS+=" -federate api_${i} ${api_pair[0]}"
39+
FEDERATED_PARAMS+=" -federationapi api_${i} ${api_pair[1]}"
40+
done
41+
42+
# To install javadoc, for example, use
43+
#
44+
# sudo apt install openjdk-11-jdk
45+
#
46+
# doclava doesn't work with openjdk-13
47+
# ```
48+
# javadoc: error - Class com.google.doclava.Doclava is not a valid doclet.
49+
# Note: As of JDK 13, the com.sun.javadoc API is no longer supported.
50+
# ```
51+
# It's used here: https://android.googlesource.com/platform/external/doclava/+/refs/heads/master/src/com/google/doclava/Doclava.java
52+
53+
# Each package in $PACKAGE needs to prefaced with -subpackages, so do that.
54+
SUBPACKAGES=""
55+
read -r -a packages <<< "${PACKAGE}"
56+
for pkg in "${packages[@]}"; do
57+
SUBPACKAGES+=" -subpackages ${pkg}"
58+
done
59+
( # Capture the return code. it may be non-zero for minor errors.
60+
/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home/bin/javadoc \
61+
-sourcepath "${SOURCE_PATH}" \
62+
-docletpath "${DOCLAVA_JAR}:${JSILVER_JAR}" \
63+
-doclet com.google.doclava.Doclava \
64+
-toroot "${SITE_PATH}"/ \
65+
-yaml _toc.yaml \
66+
-templatedir "${TEMPLATES}" \
67+
-public \
68+
-d "${OUTPUT_DIR}" \
69+
${FEDERATED_PARAMS} \
70+
${SUBPACKAGES}
71+
)
72+
73+
74+
mv "${OUTPUT_DIR}"/reference/* "${OUTPUT_DIR}"
75+
76+
###################################################################
77+
################### START OF POST-PROCESSING ######################
78+
###################################################################
79+
rm "${OUTPUT_DIR}/navtree_data.js" || true
80+
rm "${OUTPUT_DIR}/hierarchy.html" || true
81+
82+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
83+
find ${OUTPUT_DIR} -name "*.yaml" | xargs sed -i '' "s|${SITE_PATH}/reference|${SITE_PATH}|g"
84+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/org/tensorflow|a href=\"${SITE_PATH}/org/tensorflow|g"
85+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/com/google|a href=\"${SITE_PATH}/com/google|g"
86+
87+
JAVA_LANG=https://docs.oracle.com/javase/8/docs/api
88+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' "s|a href=\"reference/java/lang|a href=\"${JAVA_LANG}/java/lang|g"
89+
90+
find ${OUTPUT_DIR} -name "*.html" | xargs sed -i '' 's|<pre><code>|<pre class="prettyprint"><code>|g'
91+
92+
rm ${OUTPUT_DIR}/timestamp.js || true
93+
rm ${OUTPUT_DIR}/lists.js || true
94+
rm ${OUTPUT_DIR}/index.html || true
95+
96+
cp ${TEMPLATES}/screen.css ${OUTPUT_DIR}/
97+
98+

0 commit comments

Comments
 (0)